L1: de-literalise make_tables.py/export_stats.py; generated notes, macros (numbers.tex), prompt/regex from the snapshot, Table 12 longtable, figures (integer ticks, y from 0, per-issuer panel); stats.json never rewritten
Items: FIRC0001-AI_C-11, FIRC0001-ECON-09, FIRC0001-EDIT-17, FIRC0001-DATA-15, FIRC0001-ECON-13, FIRC0001-EDIT-14, FIRC0001-ADVE-13, FIRC0001-DATA-10, FIRC0001-EDIT-23, FIRC0001-ADVE-16, FIRC0001-AI_C-17, FIRC0001-ECON-18, FIRC0001-LITE-12, FIRC0001-DATA-01, FIRC0001-THEO-13; L2/L3 fragments: FIRC0001-ADVE-10, FIRC0001-AI_C-10, FIRC0001-DATA-06, FIRC0001-ECON-07, FIRC0001-EDIT-13, FIRC0001-LITE-08, FIRC0001-THEO-11, FIRC0001-THEO-12, FIRC0001-AI_C-15, FIRC0001-DATA-07, FIRC0001-EDIT-21, FIRC0001-AI_C-16, FIRC0001-EDIT-22, FIRC0001-EDIT-24, FIRC0001-LITE-07, FIRC0001-ADVE-06. export_stats.py edited only (run by N13 on M3U96b): run parameters from logs/ledger, new keys and dumps, read_only + threads=4. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
40 changed files +2,735 −157
modified
code/export_stats.py
+115 −16
@@ -1,24 +1,54 @@ | |||||
| 1 | """Export every number used in the working paper from the two DuckDB databases (run on the master node). | 1 | """Export every number used in the working paper from the two DuckDB databases (run on the master node). | ||
| 2 | 2 | ||||
| 3 | −Usage (on M3U96b): ~/UQO_RESEARCH_DB/.venv/bin/python export_stats.py ~/UQO_RESEARCH_DB <out_dir> | 3 | +Usage (on M3U96b): nice -n 10 env OMP_NUM_THREADS=4 ~/UQO_RESEARCH_DB/.venv/bin/python -I export_stats.py ~/UQO_RESEARCH_DB <out_dir> [--skip-slow] | ||
| 4 | −Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream. | 4 | +Writes CSV files (one per table) + stats.json. Nothing is hand-edited downstream, and no number is typed in this file: | ||
| 5 | +every value is a query result, a log/regex read (None when absent) or an explicit None flagged in the output. | ||||
| 6 | +Both connections are opened read_only=True; threads=4; nothing is written under the repository. | ||||
| 7 | +--skip-slow omits the candidate-gate scan over the whole ledger (gate_pass_total / calls_by_form) and the bootstrap. | ||||
| 5 | """ | 8 | """ | ||
| 6 | −import csv, json, sys | 9 | +import csv, json, random, re, statistics, sys | ||
| 10 | +from datetime import timedelta | ||||
| 7 | from pathlib import Path | 11 | from pathlib import Path | ||
| 8 | import duckdb | 12 | import duckdb | ||
| 9 | 13 | ||||
| 10 | −root = Path(sys.argv[1]); out = Path(sys.argv[2]); out.mkdir(parents=True, exist_ok=True) | 14 | +SEED = 20261105 | ||
| 15 | +BOOT_B = 1000 | ||||
| 16 | +args = [a for a in sys.argv[1:] if not a.startswith("--")] | ||||
| 17 | +SKIP_SLOW = "--skip-slow" in sys.argv | ||||
| 18 | +root = Path(args[0]); out = Path(args[1]); out.mkdir(parents=True, exist_ok=True) | ||||
| 11 | edgar = duckdb.connect(str(root / "data/duckdb/edgar_research.duckdb"), read_only=True) | 19 | edgar = duckdb.connect(str(root / "data/duckdb/edgar_research.duckdb"), read_only=True) | ||
| 12 | spid = duckdb.connect(str(root / "project-db/data/db/spid.duckdb"), read_only=True) | 20 | spid = duckdb.connect(str(root / "project-db/data/db/spid.duckdb"), read_only=True) | ||
| 21 | +for con in (edgar, spid): | ||||
| 22 | + con.execute("SET threads=4"); con.execute("SET memory_limit='16GB'") | ||||
| 13 | # the src_* views stored in spid.duckdb point to the master node's parquet corpus (valid here) | 23 | # the src_* views stored in spid.duckdb point to the master node's parquet corpus (valid here) | ||
| 14 | S = {} | 24 | S = {} | ||
| 15 | 25 | ||||
| 26 | + | ||||
| 16 | def dump(name, con, sql): | 27 | def dump(name, con, sql): | ||
| 17 | cur = con.execute(sql); cols = [d[0] for d in cur.description]; rows = cur.fetchall() | 28 | cur = con.execute(sql); cols = [d[0] for d in cur.description]; rows = cur.fetchall() | ||
| 18 | with open(out / f"{name}.csv", "w", newline="") as f: | 29 | with open(out / f"{name}.csv", "w", newline="") as f: | ||
| 19 | w = csv.writer(f); w.writerow(cols); w.writerows(rows) | 30 | w = csv.writer(f); w.writerow(cols); w.writerows(rows) | ||
| 20 | print(f"{name}: {len(rows)} rows"); return rows | 31 | print(f"{name}: {len(rows)} rows"); return rows | ||
| 21 | 32 | ||||
| 33 | + | ||||
| 34 | +def one(con, sql): | ||||
| 35 | + return con.execute(sql).fetchone()[0] | ||||
| 36 | + | ||||
| 37 | + | ||||
| 38 | +def grep_first(path, pattern, cast=str): | ||||
| 39 | + """First regex group found in a log file, or None when the file or the pattern is absent.""" | ||||
| 40 | + p = root / path | ||||
| 41 | + if not p.exists(): return None | ||||
| 42 | + m = re.search(pattern, p.read_text(errors="replace")) | ||||
| 43 | + return cast(m.group(1)) if m else None | ||||
| 44 | + | ||||
| 45 | + | ||||
| 46 | +def grep_count(path, pattern): | ||||
| 47 | + p = root / path | ||||
| 48 | + if not p.exists(): return None | ||||
| 49 | + return len(re.findall(pattern, p.read_text(errors="replace"))) | ||||
| 50 | + | ||||
| 51 | + | ||||
| 22 | # ---- universe reconciliation ------------------------------------------------- | 52 | # ---- universe reconciliation ------------------------------------------------- | ||
| 23 | uni = json.load(open(root / "data/metadata/universe.json")) | 53 | uni = json.load(open(root / "data/metadata/universe.json")) | ||
| 24 | ru = json.load(open(root / "data/metadata/russell3000_constituents.json")) | 54 | ru = json.load(open(root / "data/metadata/russell3000_constituents.json")) | ||
@@ -31,10 +61,15 @@ S["universe"] = {"built_at": uni["built_at"], "n": uni["count"], "n_sp500": uni[ | |||||
| 31 | "both": sum(1 for c in uni["constituents"] if c["in_sp500"] and c["in_russell3000"]), | 61 | "both": sum(1 for c in uni["constituents"] if c["in_sp500"] and c["in_russell3000"]), | ||
| 32 | "sp_only": sum(1 for c in uni["constituents"] if c["in_sp500"] and not c["in_russell3000"]), | 62 | "sp_only": sum(1 for c in uni["constituents"] if c["in_sp500"] and not c["in_russell3000"]), | ||
| 33 | "ru_only": sum(1 for c in uni["constituents"] if not c["in_sp500"] and c["in_russell3000"])} | 63 | "ru_only": sum(1 for c in uni["constituents"] if not c["in_sp500"] and c["in_russell3000"])} | ||
| 34 | −S["issuers_with_filings"] = edgar.execute("select count(distinct cik) from filings").fetchone()[0] | 64 | +S["sp500_ciks"] = len({c["cik"] for c in sp["constituents"]}) if "constituents" in sp else None | ||
| 35 | −S["issuers_without_filings"] = edgar.execute("select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)").fetchone()[0] | 65 | +# the iShares holdings file is not archived with the repository: the equity-line count is not reconstructible here | ||
| 66 | +holdings = sorted((root / "data/metadata").glob("*holding*")) | ||||
| 67 | +S["iwv_equity_lines"] = None if not holdings else sum(1 for _ in open(holdings[0])) # replaced by a parse if the file is ever archived | ||||
| 68 | +S["iwv_holdings_file_archived"] = bool(holdings) | ||||
| 69 | +S["issuers_with_filings"] = one(edgar, "select count(distinct cik) from filings") | ||||
| 70 | +S["issuers_without_filings"] = one(edgar, "select count(*) from companies c where not exists (select 1 from filings f where f.cik=c.cik)") | ||||
| 36 | dump("issuers_without_filings", edgar, "select ticker, company_name, sector, in_sp500, in_russell3000 from companies c where not exists (select 1 from filings f where f.cik=c.cik) order by ticker") | 71 | dump("issuers_without_filings", edgar, "select ticker, company_name, sector, in_sp500, in_russell3000 from companies c where not exists (select 1 from filings f where f.cik=c.cik) order by ticker") | ||
| 37 | −S["issuers_with_projects"] = spid.execute("select count(distinct cik) from projects").fetchone()[0] | 72 | +S["issuers_with_projects"] = one(spid, "select count(distinct cik) from projects") | ||
| 38 | 73 | ||||
| 39 | # ---- corpus ------------------------------------------------------------------- | 74 | # ---- corpus ------------------------------------------------------------------- | ||
| 40 | dump("filings_by_form", edgar, "select form_type, count(*) n, min(filing_date) first, max(filing_date) last, round(sum(file_size_bytes)/1e9,1) gb from filings group by 1 order by 1") | 75 | dump("filings_by_form", edgar, "select form_type, count(*) n, min(filing_date) first, max(filing_date) last, round(sum(file_size_bytes)/1e9,1) gb from filings group by 1 order by 1") | ||
@@ -44,23 +79,70 @@ dump("parse_status", edgar, "select form_type, parse_status, count(*) n from fil | |||||
| 44 | dump("sections_by_form", edgar, "select f.form_type, count(*) sections, round(avg(s.word_count)) mean_words, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) group by 1 order by 1") | 79 | dump("sections_by_form", edgar, "select f.form_type, count(*) sections, round(avg(s.word_count)) mean_words, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) group by 1 order by 1") | ||
| 45 | dump("sections_10k_by_item", edgar, "select s.section_name, count(*) n, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc") | 80 | dump("sections_10k_by_item", edgar, "select s.section_name, count(*) n, round(median(s.word_count)) median_words from filing_sections s join filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc") | ||
| 46 | S["chunks"] = edgar.execute("select count(*), round(avg(word_count),1) from filing_chunks").fetchone() | 81 | S["chunks"] = edgar.execute("select count(*), round(avg(word_count),1) from filing_chunks").fetchone() | ||
| 47 | −S["embeddings"] = edgar.execute("select count(*) from embeddings_metadata").fetchone()[0] | 82 | +S["embeddings"] = one(edgar, "select count(*) from embeddings_metadata") | ||
| 48 | −S["sections_total"] = edgar.execute("select count(*) from filing_sections").fetchone()[0] | 83 | +S["sections_total"] = one(edgar, "select count(*) from filing_sections") | ||
| 49 | −S["filings_total"] = edgar.execute("select count(*) from filings").fetchone()[0] | 84 | +S["filings_total"] = one(edgar, "select count(*) from filings") | ||
| 50 | −S["raw_gb"] = edgar.execute("select round(sum(file_size_bytes)/1e9,1) from filings").fetchone()[0] | 85 | +S["raw_gb"] = one(edgar, "select round(sum(file_size_bytes)/1e9,1) from filings") | ||
| 86 | +S["filing_date_range"] = [str(x) for x in edgar.execute("select min(filing_date), max(filing_date) from filings").fetchone()] | ||||
| 87 | +S["annual_panel_firms"] = one(edgar, "select count(distinct cik) from annual_panel") | ||||
| 51 | dump("companies_by_sector", edgar, "select sector, count(*) n, count(*) filter (where in_sp500) sp500, count(*) filter (where in_russell3000) russell from companies group by 1 order by 2 desc") | 88 | dump("companies_by_sector", edgar, "select sector, count(*) n, count(*) filter (where in_sp500) sp500, count(*) filter (where in_russell3000) russell from companies group by 1 order by 2 desc") | ||
| 52 | dump("companies_by_shard", edgar, "select shard_node, count(*) n from companies group by 1 order by 1") | 89 | dump("companies_by_shard", edgar, "select shard_node, count(*) n from companies group by 1 order by 1") | ||
| 53 | dump("factor_tables", edgar, "select 'chunk_scores' t, count(*) n from chunk_scores union all select 'filing_scores', count(*) from filing_scores union all select 'annual_panel', count(*) from annual_panel union all select 'quarterly_panel', count(*) from quarterly_panel") | 90 | dump("factor_tables", edgar, "select 'chunk_scores' t, count(*) n from chunk_scores union all select 'filing_scores', count(*) from filing_scores union all select 'annual_panel', count(*) from annual_panel union all select 'quarterly_panel', count(*) from quarterly_panel") | ||
| 54 | 91 | ||||
| 55 | # ---- extraction ledger -------------------------------------------------------- | 92 | # ---- extraction ledger -------------------------------------------------------- | ||
| 56 | dump("ledger_by_form", spid, "select v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers from extracted_sections e join v_sections v using(section_id) group by 1 order by 1") | 93 | dump("ledger_by_form", spid, "select v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers from extracted_sections e join v_sections v using(section_id) group by 1 order by 1") | ||
| 57 | −S["ledger_total"] = spid.execute("select count(*) from extracted_sections").fetchone()[0] | 94 | +S["ledger_total"] = one(spid, "select count(*) from extracted_sections") | ||
| 58 | dump("ledger_10k_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' group by 1 order by 2 desc") | 95 | dump("ledger_10k_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' group by 1 order by 2 desc") | ||
| 59 | dump("ledger_10q_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-Q' group by 1 order by 2 desc") | 96 | dump("ledger_10q_items", spid, "select v.section_name, count(*) n, sum(e.n_mentions) mentions from extracted_sections e join v_sections v using(section_id) where v.form_type='10-Q' group by 1 order by 2 desc") | ||
| 97 | +S["ledger_10k_mdna_median_words"] = one(spid, "select round(median(v.word_count)) from extracted_sections e join v_sections v using(section_id) where v.form_type='10-K' and v.section_name='md_and_a'") | ||||
| 98 | +S["ledger_backends"] = [r[0] for r in spid.execute("select distinct backend from extracted_sections order by 1").fetchall()] | ||||
| 99 | +# the two runs are separated by the backfilled block: every June row was re-inserted at one timestamp (min(extracted_at)) | ||||
| 100 | +t0, t1 = spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone() | ||||
| 101 | +S["extracted_at_range"] = [str(t0), str(t1)] | ||||
| 102 | +S["backfilled_rows"], S["backfilled_mentions"] = spid.execute("select count(*), sum(n_mentions) from extracted_sections where extracted_at = (select min(extracted_at) from extracted_sections)").fetchone() | ||||
| 103 | +S["backfilled_rows_under_30_words"] = one(spid, "select count(*) from extracted_sections e join v_sections v using(section_id) where e.extracted_at = (select min(extracted_at) from extracted_sections) and v.word_count < 30") | ||||
| 104 | +S["october_rows"] = S["ledger_total"] - S["backfilled_rows"] | ||||
| 105 | +# outage = the largest gap between consecutive distinct ledger timestamps of the October run (no window is typed here) | ||||
| 106 | +gap = spid.execute("""with t as (select distinct extracted_at ts from extracted_sections where extracted_at > (select min(extracted_at) from extracted_sections)), | ||||
| 107 | + g as (select ts, lag(ts) over (order by ts) prev from t) select prev, ts, epoch(ts - prev)/3600.0 h from g where prev is not null order by h desc limit 1""").fetchone() | ||||
| 108 | +S["outage"] = {"start": str(gap[0]), "end": str(gap[1]), "hours": round(gap[2], 2)} if gap else None | ||||
| 109 | +wall_h = (t1 - t0).total_seconds() / 3600.0 | ||||
| 110 | +S["october_wall_clock_hours"] = round(wall_h, 2) | ||||
| 111 | +S["october_effective_sections_per_second"] = round(S["october_rows"] / ((wall_h - (gap[2] if gap else 0.0)) * 3600.0), 1) | ||||
| 112 | +S["conf_exact_default_count"] = one(spid, "select count(*) from project_mentions where round(confidence,3) = 0.600") | ||||
| 113 | +S["conf_min_by_run"] = spid.execute("""select min(m.confidence) filter (where e.extracted_at = (select min(extracted_at) from extracted_sections)), | ||||
| 114 | + min(m.confidence) filter (where e.extracted_at > (select min(extracted_at) from extracted_sections)) from project_mentions m join extracted_sections e using(section_id)""").fetchone() | ||||
| 115 | +S["amount_raw_deviations"] = one(spid, "select count(*) from project_mentions where amount_usd is not null and amount_raw <> '$' || format('{:,.0f}', amount_usd)") | ||||
| 116 | +dump("ledger_collisions", spid, """select e.section_id, e.accession_number, e.n_mentions ledger_n, count(m.mention_id) stored_n, e.n_mentions - count(m.mention_id) overwritten | ||||
| 117 | + from extracted_sections e left join project_mentions m using(section_id) group by 1,2,3 having e.n_mentions > count(m.mention_id) order by 5 desc, 1""") | ||||
| 118 | +dump("cohort_by_run", spid, """select case when e.extracted_at = (select min(extracted_at) from extracted_sections) then 'backfilled (June run)' else 'October run' end run, | ||||
| 119 | + v.form_type, count(*) sections, count(*) filter (where e.n_mentions>0) with_mentions, sum(e.n_mentions) mentions, count(distinct v.cik) issuers | ||||
| 120 | + from extracted_sections e join v_sections v using(section_id) group by 1,2 order by 1,2""") | ||||
| 121 | +# run parameters are read from the logs (regex); None when the log does not carry them | ||||
| 122 | +S["spid_runs"] = { | ||||
| 123 | + "sp500_run": {"log": "project-db/logs/extract.log", "date": grep_first("project-db/logs/extract.log", r"(\d{4}-\d{2}-\d{2})"), | ||||
| 124 | + "model": grep_first("project-db/logs/extract.log", r"(gpt-4o-mini[\w-]*)"), "workers": grep_first("project-db/logs/extract.log", r"workers=(\d+)", int), | ||||
| 125 | + "batch": grep_first("project-db/logs/extract.log", r"batch=(\d+)", int), "ciks": grep_first("project-db/logs/extract.log", r"CIKs total=(\d+)", int), | ||||
| 126 | + "sections_logged": grep_first("project-db/logs/extract.log", r"sections=(\d+)", int), "ledger_rows_backfilled": S["backfilled_rows"], | ||||
| 127 | + "mentions_backfilled": S["backfilled_mentions"], "llm_call_failed": grep_count("project-db/logs/extract.log", r"LLM call failed")}, | ||||
| 128 | + "russell_run": {"log": "data/logs/spid_extract.log", "model": grep_first("data/logs/spid_extract.log", r"(gpt-4o-mini-\d{4}-\d{2}-\d{2})"), | ||||
| 129 | + "workers": grep_first("data/logs/spid_extract.log", r"workers=(\d+)", int), "ledger_rows": S["october_rows"], | ||||
| 130 | + "extracted_at_first": str(t0), "extracted_at_last": str(t1), "llm_call_failed": grep_count("data/logs/spid_extract.log", r"LLM call failed"), | ||||
| 131 | + "section_failed": grep_count("data/logs/spid_extract.log", r"section failed")}} | ||||
| 132 | +if not SKIP_SLOW: | ||||
| 133 | + # true call denominator: sections whose text passes the candidate gate (llm.py returns [] before any call otherwise) | ||||
| 134 | + gate = r"\b(project|initiative|program|invest|expan|construct|build|launch|develop|deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform|pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w*" | ||||
| 135 | + cb = dump("calls_by_form", spid, f"""select v.form_type, count(*) ledgered, count(*) filter (where regexp_matches(v.text, '{gate}', 'i')) gate_pass, | ||||
| 136 | + count(*) filter (where regexp_matches(v.text, '{gate}', 'i') and e.n_mentions>0) productive | ||||
| 137 | + from extracted_sections e join v_sections v using(section_id) group by 1 order by 1""") | ||||
| 138 | + S["gate_pass_total"] = sum(r[2] for r in cb) | ||||
| 139 | + S["calls_by_form"] = {r[0]: r[2] for r in cb} | ||||
| 140 | +else: | ||||
| 141 | + S["gate_pass_total"] = None; S["calls_by_form"] = None | ||||
| 60 | 142 | ||||
| 61 | # ---- mentions / projects ------------------------------------------------------ | 143 | # ---- mentions / projects ------------------------------------------------------ | ||
| 62 | −S["mentions"] = spid.execute("select count(*) from project_mentions").fetchone()[0] | 144 | +S["mentions"] = one(spid, "select count(*) from project_mentions") | ||
| 63 | −S["projects"] = spid.execute("select count(*) from projects").fetchone()[0] | 145 | +S["projects"] = one(spid, "select count(*) from projects") | ||
| 64 | dump("mentions_by_form", spid, "select form_type, count(*) n, round(avg(confidence),3) mean_conf from project_mentions group by 1 order by 2 desc") | 146 | dump("mentions_by_form", spid, "select form_type, count(*) n, round(avg(confidence),3) mean_conf from project_mentions group by 1 order by 2 desc") | ||
| 65 | dump("mentions_by_year", spid, "select extract(year from filing_date)::int yr, count(*) mentions, count(distinct cik) issuers from project_mentions group by 1 order by 1") | 147 | dump("mentions_by_year", spid, "select extract(year from filing_date)::int yr, count(*) mentions, count(distinct cik) issuers from project_mentions group by 1 order by 1") | ||
| 66 | dump("projects_by_type", spid, "select project_type, count(*) n, round(100.0*count(*)/(select count(*) from projects),1) pct, count(*) filter (where total_amount_usd is not null) with_amount, round(median(total_amount_usd)/1e6,1) median_amount_musd, round(avg(n_mentions),2) mean_mentions from projects group by 1 order by 2 desc") | 148 | dump("projects_by_type", spid, "select project_type, count(*) n, round(100.0*count(*)/(select count(*) from projects),1) pct, count(*) filter (where total_amount_usd is not null) with_amount, round(median(total_amount_usd)/1e6,1) median_amount_musd, round(avg(n_mentions),2) mean_mentions from projects group by 1 order by 2 desc") | ||
@@ -72,13 +154,30 @@ dump("projects_per_issuer", spid, "select quantile_cont(n, [0.1,0.25,0.5,0.75,0. | |||||
| 72 | dump("projects_by_index", spid, "select co.in_sp500, co.in_russell3000, count(*) projects, count(distinct p.cik) issuers from projects p join src_companies co on p.cik=co.cik group by 1,2") | 154 | dump("projects_by_index", spid, "select co.in_sp500, co.in_russell3000, count(*) projects, count(distinct p.cik) issuers from projects p join src_companies co on p.cik=co.cik group by 1,2") | ||
| 73 | dump("confidence_dist", spid, "select round(confidence,1) conf, count(*) n from project_mentions group by 1 order by 1") | 155 | dump("confidence_dist", spid, "select round(confidence,1) conf, count(*) n from project_mentions group by 1 order by 1") | ||
| 74 | dump("amount_dist", spid, "select quantile_cont(total_amount_usd, [0.1,0.25,0.5,0.75,0.9,0.99]) q, count(*) n, round(sum(total_amount_usd)/1e9,1) sum_gusd from projects where total_amount_usd is not null") | 156 | dump("amount_dist", spid, "select quantile_cont(total_amount_usd, [0.1,0.25,0.5,0.75,0.9,0.99]) q, count(*) n, round(sum(total_amount_usd)/1e9,1) sum_gusd from projects where total_amount_usd is not null") | ||
| 157 | +S["amount_total_median_musd"] = one(spid, "select round(median(total_amount_usd)/1e6,1) from projects where total_amount_usd is not null") | ||||
| 158 | +S["amount_max"] = one(spid, "select max(total_amount_usd) from projects") | ||||
| 159 | +S["amount_ge_50b"] = one(spid, "select count(*) from projects where total_amount_usd >= 50e9") | ||||
| 160 | +S["amount_ge_1t"] = one(spid, "select count(*) from projects where total_amount_usd >= 1e12") | ||||
| 161 | +S["amount_le_zero"] = one(spid, "select count(*) from projects where total_amount_usd <= 0") | ||||
| 75 | dump("duration", spid, "select case when last_year is null or first_year is null then 'unknown' when last_year=first_year then '0' when last_year-first_year<=2 then '1-2' when last_year-first_year<=5 then '3-5' else '>5' end span_years, count(*) n from projects group by 1 order by 1") | 162 | dump("duration", spid, "select case when last_year is null or first_year is null then 'unknown' when last_year=first_year then '0' when last_year-first_year<=2 then '1-2' when last_year-first_year<=5 then '3-5' else '>5' end span_years, count(*) n from projects group by 1 order by 1") | ||
| 76 | dump("kg", spid, "select 'nodes' k, node_type, count(*) n from kg_nodes group by 1,2 union all select 'edges', rel, count(*) from kg_edges group by 1,2 order by 1,3 desc") | 163 | dump("kg", spid, "select 'nodes' k, node_type, count(*) n from kg_nodes group by 1,2 union all select 'edges', rel, count(*) from kg_edges group by 1,2 order by 1,3 desc") | ||
| 77 | dump("top_issuers", spid, "select ticker, company_name, sector, count(*) projects from projects group by 1,2,3 order by 4 desc limit 15") | 164 | dump("top_issuers", spid, "select ticker, company_name, sector, count(*) projects from projects group by 1,2,3 order by 4 desc limit 15") | ||
| 78 | dump("example_provenance", spid, "select p.project_id, p.ticker, p.project_name, p.project_type, p.status, p.total_amount_usd, p.n_mentions, p.first_seen, p.last_seen, t.accession_number, t.filing_date, t.form_type, t.status m_status, t.amount_usd, t.confidence, t.snippet from projects p join project_timeline t using(project_id) where p.n_mentions between 4 and 8 and p.total_amount_usd > 5e8 and p.project_type in ('plant_construction','data_center') order by p.n_filings desc, p.project_id, t.filing_date limit 24") | 165 | dump("example_provenance", spid, "select p.project_id, p.ticker, p.project_name, p.project_type, p.status, p.total_amount_usd, p.n_mentions, p.first_seen, p.last_seen, t.accession_number, t.filing_date, t.form_type, t.status m_status, t.amount_usd, t.confidence, t.snippet from projects p join project_timeline t using(project_id) where p.n_mentions between 4 and 8 and p.total_amount_usd > 5e8 and p.project_type in ('plant_construction','data_center') order by p.n_filings desc, p.project_id, t.filing_date limit 24") | ||
| 79 | S["mentions_fields"] = spid.execute("select count(*) filter (where amount_usd is not null) amt, count(*) filter (where len(locations)>0) loc, count(*) filter (where len(technologies)>0) tech, count(*) filter (where len(partners)>0) partner, count(*) filter (where len(years)>0) yrs, count(*) from project_mentions").fetchone() | 166 | S["mentions_fields"] = spid.execute("select count(*) filter (where amount_usd is not null) amt, count(*) filter (where len(locations)>0) loc, count(*) filter (where len(technologies)>0) tech, count(*) filter (where len(partners)>0) partner, count(*) filter (where len(years)>0) yrs, count(*) from project_mentions").fetchone() | ||
| 80 | S["projects_fields"] = spid.execute("select count(*) filter (where total_amount_usd is not null), count(*) filter (where canonical_location is not null), count(*) filter (where len(technologies)>0), count(*) from projects").fetchone() | 167 | S["projects_fields"] = spid.execute("select count(*) filter (where total_amount_usd is not null), count(*) filter (where canonical_location is not null), count(*) filter (where len(technologies)>0), count(*) from projects").fetchone() | ||
| 81 | −S["spid_runs"] = {"sp500_run": "2026-06 (gpt-4o-mini, 16 workers, 153,894 sections)", "russell_run": "2026-10-27/28 (gpt-4o-mini-2024-07-18, 48 workers, 968,134 sections)"} | 168 | +S["fiscal_year_null_mentions"] = one(spid, "select count(*) from project_mentions where fiscal_year is null") | ||
| 82 | −S["extracted_at_range"] = [str(x) for x in spid.execute("select min(extracted_at), max(extracted_at) from extracted_sections").fetchone()] | 169 | + | ||
| 170 | +# ---- bootstrap CI of the per-type median amount (percentile bootstrap, seeded) ---- | ||||
| 171 | +if not SKIP_SLOW: | ||||
| 172 | + rng = random.Random(SEED) | ||||
| 173 | + with open(out / "projects_by_type_ci.csv", "w", newline="") as f: | ||||
| 174 | + w = csv.writer(f); w.writerow(["project_type", "n", "median", "ci_low", "ci_high", "B", "seed"]) | ||||
| 175 | + for ptype, in spid.execute("select project_type from projects group by 1 order by count(*) desc").fetchall(): | ||||
| 176 | + vals = [r[0] for r in spid.execute("select total_amount_usd from projects where total_amount_usd is not null and project_type = ?", [ptype]).fetchall()] | ||||
| 177 | + if not vals: w.writerow([ptype, 0, None, None, None, BOOT_B, SEED]); continue | ||||
| 178 | + meds = sorted(statistics.median(rng.choices(vals, k=len(vals))) for _ in range(BOOT_B)) | ||||
| 179 | + w.writerow([ptype, len(vals), statistics.median(vals), meds[int(0.025 * BOOT_B)], meds[min(int(0.975 * BOOT_B), BOOT_B - 1)], BOOT_B, SEED]) | ||||
| 180 | + print("projects_by_type_ci: written") | ||||
| 181 | + | ||||
| 83 | json.dump(S, open(out / "stats.json", "w"), indent=2, default=str) | 182 | json.dump(S, open(out / "stats.json", "w"), indent=2, default=str) | ||
| 84 | print(json.dumps(S, indent=1, default=str)[:3000]) | 183 | print(json.dumps(S, indent=1, default=str)[:3000]) | ||
modified
code/make_tables.py
+573 −61
@@ -1,25 +1,82 @@ | |||||
| 1 | −"""Build every LaTeX table and figure of the paper from data/*.csv (no hand-typed numbers).""" | 1 | +"""Build every LaTeX table, figure and inline-number macro of the paper from the exported data. | ||
| 2 | −import csv, json | 2 | + | ||
| 3 | +Inputs (read-only): | ||||
| 4 | + data/*.csv, data/stats.json exports of code/export_stats.py (node, DuckDB read-only) | ||||
| 5 | + data/documented_inputs.json the few values that no query can reproduce (each with its source) | ||||
| 6 | + data/datamodel_audit.csv rows of the data-model audit table (Appendix C) | ||||
| 7 | + code/audit/spid_snapshot/{llm,rules,taxonomy}.py frozen pipeline snapshot (prompt, gate regex, taxonomy) | ||||
| 8 | + results/**/*.csv Stage D audit outputs (optional; tables are skipped when absent) | ||||
| 9 | +Outputs: | ||||
| 10 | + paper/tables/*.tex, paper/figures/*.pdf, paper/tables/numbers.tex (SPID... macros for inline numbers), | ||||
| 11 | + results/inline_numbers.json (macro -> value, source, formula), data/example_project.json. | ||||
| 12 | +No numeric literal is typed in this file; stats.json is never rewritten. | ||||
| 13 | +Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) | ||||
| 14 | +""" | ||||
| 15 | +import argparse, ast, csv, json, math, re, sys | ||||
| 3 | from pathlib import Path | 16 | from pathlib import Path | ||
| 4 | import matplotlib | 17 | import matplotlib | ||
| 5 | matplotlib.use("Agg") | 18 | matplotlib.use("Agg") | ||
| 6 | import matplotlib.pyplot as plt | 19 | import matplotlib.pyplot as plt | ||
| 20 | +from matplotlib.ticker import MaxNLocator | ||||
| 7 | 21 | ||||
| 8 | ROOT = Path(__file__).resolve().parent.parent | 22 | ROOT = Path(__file__).resolve().parent.parent | ||
| 9 | −D, T, F = ROOT / "data", ROOT / "paper" / "tables", ROOT / "paper" / "figures" | 23 | +ap = argparse.ArgumentParser() | ||
| 10 | −T.mkdir(exist_ok=True); F.mkdir(exist_ok=True) | 24 | +ap.add_argument("--out", default=None, help="scratch root; tables/, figures/, results/ are created under it (tests)") | ||
| 11 | −S = json.load(open(D / "stats.json")) | 25 | +ARGS = ap.parse_args() | ||
| 26 | +D = ROOT / "data" | ||||
| 27 | +R = ROOT / "results" | ||||
| 28 | +SNAP = ROOT / "code" / "audit" / "spid_snapshot" | ||||
| 29 | +OUT = Path(ARGS.out) if ARGS.out else ROOT / "paper" | ||||
| 30 | +T, F = OUT / "tables", OUT / "figures" | ||||
| 31 | +RES_OUT = (Path(ARGS.out) / "results") if ARGS.out else R | ||||
| 32 | +DATA_OUT = (Path(ARGS.out) / "data") if ARGS.out else D | ||||
| 33 | +for p in (T, F, RES_OUT, DATA_OUT): | ||||
| 34 | + p.mkdir(parents=True, exist_ok=True) | ||||
| 35 | + | ||||
| 36 | +S = json.load(open(D / "stats.json")) # read-only | ||||
| 37 | +DI = json.load(open(D / "documented_inputs.json")) if (D / "documented_inputs.json").exists() else {} | ||||
| 38 | +INLINE = {} # macro -> {value, source, formula} | ||||
| 12 | 39 | ||||
| 13 | −def rows(name): | 40 | + | ||
| 14 | − with open(D / f"{name}.csv") as f: | 41 | +def rows(name, base=D): | ||
| 42 | + with open(base / f"{name}.csv", newline="") as f: | ||||
| 15 | return list(csv.DictReader(f)) | 43 | return list(csv.DictReader(f)) | ||
| 16 | 44 | ||||
| 45 | + | ||||
| 17 | def n(x, d=0): | 46 | def n(x, d=0): | ||
| 18 | x = float(x) | 47 | x = float(x) | ||
| 19 | return f"{x:,.{d}f}".replace(",", "{,}") | 48 | return f"{x:,.{d}f}".replace(",", "{,}") | ||
| 20 | 49 | ||||
| 50 | + | ||||
| 21 | def w(name, body): | 51 | def w(name, body): | ||
| 22 | − (T / name).write_text(body); print("table", name) | 52 | + (T / name).write_text(body if body.endswith("\n") else body + "\n") | ||
| 53 | + print("table", name) | ||||
| 54 | + | ||||
| 55 | + | ||||
| 56 | +def esc(s): | ||||
| 57 | + return (str(s).replace("\\", "\\textbackslash{}").replace("&", "\\&").replace("%", "\\%") | ||||
| 58 | + .replace("$", "\\$").replace("_", "\\_").replace("#", "\\#")) | ||||
| 59 | + | ||||
| 60 | + | ||||
| 61 | +_DIGITS = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"} | ||||
| 62 | + | ||||
| 63 | + | ||||
| 64 | +def macro(name, value, source, formula): | ||||
| 65 | + """Register an inline number: \\SPID<name> in paper/tables/numbers.tex (control sequences allow letters only).""" | ||||
| 66 | + name = "".join(_DIGITS.get(ch, ch) for ch in name if ch.isalnum()) | ||||
| 67 | + INLINE[name] = {"value": value, "source": source, "formula": formula} | ||||
| 68 | + | ||||
| 69 | + | ||||
| 70 | +def di(key): | ||||
| 71 | + """Documented input (value + source) or None.""" | ||||
| 72 | + e = DI.get(key) | ||||
| 73 | + return None if e is None else e.get("value") | ||||
| 74 | + | ||||
| 75 | + | ||||
| 76 | +def di_source(key): | ||||
| 77 | + e = DI.get(key) | ||||
| 78 | + return "" if e is None else e.get("source", "") | ||||
| 79 | + | ||||
| 23 | 80 | ||||
| 24 | LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", | 81 | LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expenditure program", "partnership": "Strategic partnership", | ||
| 25 | "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", | 82 | "geographic_expansion": "Geographic expansion", "plant_construction": "Plant / site construction", "cost_reduction": "Cost reduction / restructuring", | ||
@@ -29,16 +86,30 @@ LAB = {"ma_integration": "M\\&A / integration", "capex_program": "Capital expend | |||||
| 29 | "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", | 86 | "sustainability": "Sustainability / ESG", "supply_chain": "Supply-chain initiative", "data_center": "Data center", "energy_transition": "Energy transition / renewables", | ||
| 30 | "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} | 87 | "ai_initiative": "AI / ML initiative", "automation": "Automation / robotics", "cloud_migration": "Cloud migration"} | ||
| 31 | 88 | ||||
| 32 | −# ---- Table: universe reconciliation ---- | 89 | +# ---- Table 1: universe reconciliation ---------------------------------------- | ||
| 33 | U = S["universe"] | 90 | U = S["universe"] | ||
| 91 | +iwv_lines = S.get("iwv_equity_lines") # exported by export_stats.py when the holdings file is archived | ||||
| 92 | +iwv_src = "stats.json:iwv_equity_lines (export_stats.py)" | ||||
| 93 | +if iwv_lines is None: | ||||
| 94 | + iwv_lines = di("iwv_equity_lines_reported"); iwv_src = di_source("iwv_equity_lines_reported") | ||||
| 95 | +if iwv_lines is not None: | ||||
| 96 | + row1_label = (f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); " | ||||
| 97 | + "count reported by the build log, source file not archived" if "not archived" in iwv_src | ||||
| 98 | + else f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']})") | ||||
| 99 | + row1_val = n(iwv_lines) | ||||
| 100 | +else: | ||||
| 101 | + row1_label = f"iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}); source file not archived" | ||||
| 102 | + row1_val = "--" | ||||
| 103 | +macro("iwvEquityLines", row1_val, iwv_src, "documented input or stats.json") | ||||
| 104 | +sp500_ciks = S.get("sp500_ciks", U.get("n_sp500")) | ||||
| 34 | w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}} | 105 | w("tab_reconciliation.tex", f"""\\begin{{tabular}}{{lr}} | ||
| 35 | \\toprule | 106 | \\toprule | ||
| 36 | Step & Count \\\\ | 107 | Step & Count \\\\ | ||
| 37 | \\midrule | 108 | \\midrule | ||
| 38 | −iShares IWV holdings file, equity lines with a ticker (as of {U['russell_holdings_as_of']}) & 2{{,}}587 \\\\ | 109 | +{row1_label} & {row1_val} \\\\ | ||
| 39 | \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ | 110 | \\quad of which mapped to an SEC CIK (CVRs, escrow lines and FDIC-only banks excluded) & {n(U['russell_equity_lines_mapped'])} \\\\ | ||
| 40 | \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ | 111 | \\quad distinct CIKs (share classes collapsed) & {n(U['russell_distinct_ciks'])} \\\\ | ||
| 41 | −S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} (500 CIKs) & {n(U['sp500_rows'])} \\\\ | 112 | +S\\&P~500 constituents, cached snapshot of {U['sp500_fetched_at'][:10]} ({n(sp500_ciks)} CIKs) & {n(U['sp500_rows'])} \\\\ | ||
| 42 | \\midrule | 113 | \\midrule | ||
| 43 | Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ | 114 | Merged issuer universe (distinct CIKs) & {n(U['n'])} \\\\ | ||
| 44 | \\quad in both lists & {n(U['both'])} \\\\ | 115 | \\quad in both lists & {n(U['both'])} \\\\ | ||
@@ -50,85 +121,288 @@ Issuers without any such filing (foreign private issuers filing 20-F/40-F, FDIC- | |||||
| 50 | Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ | 121 | Issuers with at least one resolved project & {n(S['issuers_with_projects'])} \\\\ | ||
| 51 | \\bottomrule | 122 | \\bottomrule | ||
| 52 | \\end{{tabular}}""") | 123 | \\end{{tabular}}""") | ||
| 124 | +macro("universeN", n(U["n"]), "stats.json:universe.n", "count") | ||||
| 125 | +macro("russellDistinctCiks", n(U["russell_distinct_ciks"]), "stats.json:universe.russell_distinct_ciks", "count") | ||||
| 126 | +macro("russellMappedLines", n(U["russell_equity_lines_mapped"]), "stats.json:universe.russell_equity_lines_mapped", "count") | ||||
| 127 | +macro("russellUnmatched", n(U["russell_unmatched"]), "stats.json:universe.russell_unmatched", "len(unmatched)") | ||||
| 128 | +# Russell 3000 coverage of the proxy (L5): only if results/russell_coverage.json holds a retrieved constituent count | ||||
| 129 | +rc_path = R / "russell_coverage.json" | ||||
| 130 | +if rc_path.exists(): | ||||
| 131 | + rc = json.load(open(rc_path)) | ||||
| 132 | + if rc.get("index_constituents") is not None: | ||||
| 133 | + cov = 100.0 * U["russell_distinct_ciks"] / rc["index_constituents"] | ||||
| 134 | + macro("russellIndexConstituents", n(rc["index_constituents"]), "results/russell_coverage.json:index_constituents", rc.get("source_url", "")) | ||||
| 135 | + macro("russellProxyCoveragePct", f"{cov:.1f}", "results/russell_coverage.json", "100 * russell_distinct_ciks / index_constituents") | ||||
| 136 | + if rc.get("fund_holdings_count") is not None: | ||||
| 137 | + macro("iwvHoldingsCount", n(rc["fund_holdings_count"]), "results/russell_coverage.json:fund_holdings_count", rc.get("fund_source_url", "")) | ||||
| 138 | + if rc.get("replication_method"): | ||||
| 139 | + macro("iwvReplicationMethod", esc(rc["replication_method"]), "results/russell_coverage.json:replication_method", rc.get("fund_source_url", "")) | ||||
| 53 | 140 | ||||
| 54 | −# ---- Table: filings by form ---- | 141 | +# ---- Table 2: filings by form -------------------------------------------------- | ||
| 55 | fb = rows("filings_by_form"); ps = rows("parse_status") | 142 | fb = rows("filings_by_form"); ps = rows("parse_status") | ||
| 56 | err = {} | 143 | err = {} | ||
| 57 | for r in ps: | 144 | for r in ps: | ||
| 58 | err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) | 145 | err.setdefault(r["form_type"], {})[r["parse_status"]] = int(r["n"]) | ||
| 146 | +fdr = S.get("filing_date_range") or [min(r["first"] for r in fb), max(r["last"] for r in fb)] | ||||
| 147 | +macro("filingDateFirst", fdr[0], "stats.json:filing_date_range or min(filings_by_form.first)", "min") | ||||
| 148 | +macro("filingDateLast", fdr[1], "stats.json:filing_date_range or max(filings_by_form.last)", "max") | ||||
| 59 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n" | 149 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nForm & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\\\\n\\midrule\n" | ||
| 60 | for r in fb: | 150 | for r in fb: | ||
| 61 | e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0) | 151 | e = err.get(r["form_type"], {}); notp = e.get("error", 0) + e.get("missing", 0) | ||
| 62 | body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n" | 152 | body += f"{r['form_type']} & {n(r['n'])} & {r['first']} & {r['last']} & {float(r['gb']):.1f} & {notp} \\\\\n" | ||
| 63 | −body += f"\\midrule\nTotal & {n(S['filings_total'])} & 2010-01-04 & 2026-10-27 & {S['raw_gb']:.1f} & {sum(e.get('error',0)+e.get('missing',0) for e in err.values())} \\\\\n\\bottomrule\n\\end{{tabular}}" | 153 | +not_parsed = sum(e.get("error", 0) + e.get("missing", 0) for e in err.values()) | ||
| 154 | +missing = sum(e.get("missing", 0) for e in err.values()) | ||||
| 155 | +body += f"\\midrule\nTotal & {n(S['filings_total'])} & {fdr[0]} & {fdr[1]} & {S['raw_gb']:.1f} & {not_parsed} \\\\\n\\bottomrule\n\\end{{tabular}}" | ||||
| 64 | w("tab_filings.tex", body) | 156 | w("tab_filings.tex", body) | ||
| 157 | +macro("filingsTotal", n(S["filings_total"]), "stats.json:filings_total", "count") | ||||
| 158 | +macro("filingsNotParsed", str(not_parsed), "data/parse_status.csv", "sum(error)+sum(missing)") | ||||
| 159 | +macro("filingsMissingOnDisk", str(missing), "data/parse_status.csv", "sum(missing)") | ||||
| 160 | +macro("rawGb", f"{S['raw_gb']:.1f}", "stats.json:raw_gb", "sum(file_size_bytes)/1e9") | ||||
| 65 | 161 | ||||
| 66 | −# ---- Table: coverage by year ---- | 162 | +# ---- Table: coverage by year ---------------------------------------------------- | ||
| 67 | −iy = rows("issuers_by_year"); fy = rows("filings_by_year_form"); py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | 163 | +iy = rows("issuers_by_year"); fy = rows("filings_by_year_form") | ||
| 164 | +py = {r["yr"]: r for r in rows("projects_by_first_seen_year")}; my = {r["yr"]: r for r in rows("mentions_by_year")} | ||||
| 68 | byf = {} | 165 | byf = {} | ||
| 69 | for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] | 166 | for r in fy: byf.setdefault(r["yr"], {})[r["form_type"]] = r["n"] | ||
| 70 | −body = "\\begin{tabular}{lrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\\\\n\\midrule\n" | 167 | +body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nYear & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\\\\n\\midrule\n" | ||
| 71 | for r in iy: | 168 | for r in iy: | ||
| 72 | − y = r["yr"]; b = byf[y] | 169 | + y = r["yr"]; b = byf[y]; mpi = int(my[y]["mentions"]) / int(my[y]["issuers"]) | ||
| 73 | − body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {n(py[y]['n'])} \\\\\n" | 170 | + body += f"{y} & {n(r['issuers'])} & {n(b['10-K'])} & {n(b['10-Q'])} & {n(b['8-K'])} & {n(my[y]['mentions'])} & {n(my[y]['issuers'])} & {mpi:.1f} & {n(py[y]['n'])} \\\\\n" | ||
| 74 | body += "\\bottomrule\n\\end{tabular}" | 171 | body += "\\bottomrule\n\\end{tabular}" | ||
| 75 | w("tab_coverage_year.tex", body) | 172 | w("tab_coverage_year.tex", body) | ||
| 173 | +yrs_all = sorted(int(r["yr"]) for r in iy) | ||||
| 174 | +y_first, y_last = yrs_all[0], yrs_all[-1] | ||||
| 175 | +inner = [y for y in yrs_all if y_first < y < y_last] # exclude the left-censored first year and the partial last year | ||||
| 176 | +fs_inner = [int(py[str(y)]["n"]) for y in inner] | ||||
| 177 | +macro("firstSeenYearMin", str(inner[0]), "data/projects_by_first_seen_year.csv", "first full year after the left-censored start") | ||||
| 178 | +macro("firstSeenYearMax", str(inner[-1]), "data/projects_by_first_seen_year.csv", "last full year before the partial final year") | ||||
| 179 | +macro("firstSeenRangeInner", f"{n(min(fs_inner))}--{n(max(fs_inner))}", "data/projects_by_first_seen_year.csv", f"min/max of projects first seen, {inner[0]}--{inner[-1]}") | ||||
| 180 | +macro("firstSeenFirstYear", n(py[str(y_first)]["n"]), "data/projects_by_first_seen_year.csv", f"projects first seen in {y_first} (left-censored)") | ||||
| 181 | +macro("firstSeenInnerMean", n(sum(fs_inner) / len(fs_inner)), "data/projects_by_first_seen_year.csv", f"mean {inner[0]}--{inner[-1]}") | ||||
| 182 | +macro("mentionsFirstYear", n(my[str(y_first)]["mentions"]), "data/mentions_by_year.csv", f"mentions in {y_first}") | ||||
| 183 | +macro("mentionsLastFullYear", n(my[str(inner[-1])]["mentions"]), "data/mentions_by_year.csv", f"mentions in {inner[-1]}") | ||||
| 184 | +macro("issuersWithMentionsFirstYear", n(my[str(y_first)]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {y_first}") | ||||
| 185 | +macro("issuersWithMentionsLastFullYear", n(my[str(inner[-1])]["issuers"]), "data/mentions_by_year.csv", f"issuers with mentions in {inner[-1]}") | ||||
| 186 | +macro("mentionsPerIssuerFirstYear", f"{int(my[str(y_first)]['mentions'])/int(my[str(y_first)]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | ||||
| 187 | +macro("mentionsPerIssuerLastFullYear", f"{int(my[str(inner[-1])]['mentions'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/mentions_by_year.csv", "mentions/issuers") | ||||
| 188 | +macro("lastYear", str(y_last), "data/issuers_by_year.csv", "max year (partial)") | ||||
| 76 | 189 | ||||
| 77 | −# ---- Table: sections and extraction ledger ---- | 190 | +# ---- Table 3: sections and extraction ledger ----------------------------------- | ||
| 78 | sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") | 191 | sb = {r["form_type"]: r for r in rows("sections_by_form")}; lb = rows("ledger_by_form") | ||
| 79 | −body = "\\begin{tabular}{lrrrrrr}\n\\toprule\nForm & Parsed sections & Median words & Sections sent to the extractor & Sections with $\\geq 1$ mention & Mentions & Issuers \\\\\n\\midrule\n" | 192 | +mf = {r["form_type"]: int(r["n"]) for r in rows("mentions_by_form")} | ||
| 80 | −tot = [0, 0, 0, 0] | 193 | +calls = {} | ||
| 194 | +t3 = R / "ledger" / "table3_ledger_vs_stored.csv" | ||||
| 195 | +if t3.exists(): | ||||
| 196 | + calls = {r["form_type"]: int(float(r["calls"])) for r in rows(t3.stem, t3.parent) if r.get("calls")} | ||||
| 197 | +cc = " & Gate-passing calls" if calls else "" | ||||
| 198 | +body = ("\\begin{tabular}{lrrrr" + ("r" if calls else "") + "rrr}\n\\toprule\nForm & Parsed sections & Median words & Ledgered sections ($\\geq$ 30 words)" + cc + " & " | ||||
| 199 | + "Sections with $\\geq 1$ mention & Mentions returned by the model (ledger) & Mentions stored & Issuers \\\\\n\\midrule\n") | ||||
| 200 | +tot = [0, 0, 0, 0, 0]; gap_by_form = {} | ||||
| 81 | for r in lb: | 201 | for r in lb: | ||
| 82 | − s = sb[r["form_type"]] | 202 | + s = sb[r["form_type"]]; stored = mf[r["form_type"]] | ||
| 83 | − body += f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])} & {n(r['with_mentions'])} & {n(r['mentions'])} & {n(r['issuers'])} \\\\\n" | 203 | + gap_by_form[r["form_type"]] = int(r["mentions"]) - stored | ||
| 84 | − tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]) | 204 | + cv = f" & {n(calls[r['form_type']])}" if calls else "" | ||
| 85 | −body += f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])} & {n(tot[2])} & {n(tot[3])} & {n(S['issuers_with_filings'])} \\\\\n\\bottomrule\n\\end{{tabular}}" | 205 | + body += (f"{r['form_type']} & {n(s['sections'])} & {n(s['median_words'])} & {n(r['sections'])}{cv} & {n(r['with_mentions'])} & " | ||
| 206 | + f"{n(r['mentions'])} & {n(stored)} & {n(r['issuers'])} \\\\\n") | ||||
| 207 | + tot[0] += int(s["sections"]); tot[1] += int(r["sections"]); tot[2] += int(r["with_mentions"]); tot[3] += int(r["mentions"]); tot[4] += stored | ||||
| 208 | +ctot = f" & {n(sum(calls.values()))}" if calls else "" | ||||
| 209 | +body += (f"\\midrule\nTotal & {n(tot[0])} & & {n(tot[1])}{ctot} & {n(tot[2])} & {n(tot[3])} & {n(tot[4])} & {n(S['issuers_with_filings'])}$^{{\\dagger}}$ \\\\\n" | ||||
| 210 | + "\\bottomrule\n\\end{tabular}") | ||||
| 211 | +if calls: | ||||
| 212 | + for ft, v in calls.items(): | ||||
| 213 | + macro(f"calls{ft}", n(v), "results/ledger/table3_ledger_vs_stored.csv", f"calls where form_type={ft}") | ||||
| 86 | w("tab_sections.tex", body) | 214 | w("tab_sections.tex", body) | ||
| 215 | +ledger_gap = tot[3] - tot[4] | ||||
| 216 | +gap_txt = ", ".join(f"{k} {v}" for k, v in gap_by_form.items()) | ||||
| 217 | +coll_sections = None | ||||
| 218 | +for cand in (R / "ledger" / "ledger_collisions.csv", D / "ledger_collisions.csv"): | ||||
| 219 | + if cand.exists(): | ||||
| 220 | + with open(cand, newline="") as f: | ||||
| 221 | + coll_sections = sum(1 for _ in csv.DictReader(f)) | ||||
| 222 | + break | ||||
| 223 | +note = (f"\\item ``Ledgered sections'' counts rows of the \\texttt{{extracted\\_sections}} ledger (sections with at least 30 words; " | ||||
| 224 | + f"for 10-Ks restricted to the items of Table~\\ref{{tab:items}}); the ledger does not record whether the candidate gate " | ||||
| 225 | + f"passed or whether the API call succeeded, so this is not a count of calls" | ||||
| 226 | + + (f"; ``Gate-passing calls'' recomputes the candidate gate of Appendix~\\ref{{app:prompt}} over the ledgered text ({n(sum(calls.values()))} sections reached the model)" if calls else "") | ||||
| 227 | + + ". ``Mentions returned by the model'' is " | ||||
| 228 | + f"$\\sum$\\texttt{{n\\_mentions}} over the ledger ({n(tot[3])}); ``Mentions stored'' is the row count of " | ||||
| 229 | + f"\\texttt{{project\\_mentions}} ({n(tot[4])}). The difference of {n(ledger_gap)} ({gap_txt})" | ||||
| 230 | + + (f", spread over {n(coll_sections)} sections," if coll_sections else "") | ||||
| 231 | + + " is the number of mentions overwritten by mention-identifier collisions before storage (Section~\\ref{sec:construct}). " | ||||
| 232 | + f"$^{{\\dagger}}$Issuers with at least one filing of any form, not a column sum.\n") | ||||
| 233 | +w("tab_sections_note.tex", note) | ||||
| 234 | +macro("ledgerTotal", n(S["ledger_total"]), "stats.json:ledger_total", "count(extracted_sections)") | ||||
| 235 | +macro("ledgerMentionsReturned", n(tot[3]), "data/ledger_by_form.csv", "sum(mentions)") | ||||
| 236 | +macro("ledgerGap", n(ledger_gap), "data/ledger_by_form.csv, data/mentions_by_form.csv", "sum(ledger mentions) - sum(stored mentions)") | ||||
| 237 | +macro("ledgerGapByForm", gap_txt, "data/ledger_by_form.csv, data/mentions_by_form.csv", "per form") | ||||
| 238 | +for ft in ("8-K", "10-K", "10-Q"): | ||||
| 239 | + key = ft.replace("-", "") | ||||
| 240 | + macro(f"mentionsStored{key}", n(mf[ft]), "data/mentions_by_form.csv", f"n where form_type={ft}") | ||||
| 241 | + lr = next(r for r in lb if r["form_type"] == ft) | ||||
| 242 | + macro(f"mentionsLedger{key}", n(lr["mentions"]), "data/ledger_by_form.csv", f"mentions where form_type={ft}") | ||||
| 243 | + macro(f"ledgerSections{key}", n(lr["sections"]), "data/ledger_by_form.csv", f"sections where form_type={ft}") | ||||
| 244 | +productive = tot[2] | ||||
| 245 | +macro("productiveSections", n(productive), "data/ledger_by_form.csv", "sum(with_mentions)") | ||||
| 246 | +macro("productiveShareLedger", f"{100*productive/S['ledger_total']:.1f}", "data/ledger_by_form.csv, stats.json", "100*productive/ledger_total") | ||||
| 247 | +macro("mentionsPerLedgerSection", f"{S['mentions']/S['ledger_total']:.2f}", "stats.json", "mentions/ledger_total") | ||||
| 248 | +macro("mentionsPerProductiveSection", f"{S['mentions']/productive:.2f}", "stats.json, data/ledger_by_form.csv", "mentions/productive") | ||||
| 249 | +gate_total = S.get("gate_pass_total") | ||||
| 250 | +gt_path = R / "ledger" / "gate_pass_totals.json" | ||||
| 251 | +if not gate_total and gt_path.exists(): | ||||
| 252 | + gt = json.load(open(gt_path)) | ||||
| 253 | + gate_total = next((gt[k] for k in ("calls", "gate_pass", "calls_total", "gate_pass_total") if isinstance(gt.get(k), (int, float))), None) | ||||
| 254 | +if gate_total: | ||||
| 255 | + g = gate_total | ||||
| 256 | + S = dict(S); S["gate_pass_total"] = g # local copy only; stats.json is never written | ||||
| 257 | + macro("gatePassTotal", n(g), "stats.json:gate_pass_total", "regexp_matches(_PROJECT_GATE) over the ledger") | ||||
| 258 | + macro("productiveShareCalls", f"{100*productive/g:.1f}", "stats.json:gate_pass_total", "100*productive/gate_pass_total") | ||||
| 259 | + macro("mentionsPerCall", f"{S['mentions']/g:.2f}", "stats.json:gate_pass_total", "mentions/gate_pass_total") | ||||
| 87 | 260 | ||||
| 88 | −# ---- Table: 10-K / 10-Q items in scope ---- | 261 | +# ---- Table 4: 10-K / 10-Q items in scope ---------------------------------------- | ||
| 89 | −body = "\\begin{tabular}{llrr}\n\\toprule\nForm & Section (item) & Sections extracted & Mentions \\\\\n\\midrule\n" | 262 | +gp = None | ||
| 90 | −for r in rows("ledger_10k_items"): body += f"10-K & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | 263 | +for cand in (R / "ledger" / "gate_pass_by_form_item.csv",): | ||
| 91 | −body += "\\midrule\n" | 264 | + if cand.exists(): | ||
| 92 | −for r in rows("ledger_10q_items"): body += f"10-Q & \\texttt{{{r['section_name'].replace('_','\\_')}}} & {n(r['n'])} & {n(r['mentions'])} \\\\\n" | 265 | + gp = {} | ||
| 266 | + for r in rows(cand.stem, cand.parent): | ||||
| 267 | + fk = next((r[c] for c in r if c in ("form", "form_type")), None) | ||||
| 268 | + ik = next((r[c] for c in r if c in ("item", "section_name")), None) | ||||
| 269 | + gk = r.get("calls", next((r[c] for c in r if "gate_pass" in c.lower()), None)) | ||||
| 270 | + if fk and ik and gk not in (None, ""): | ||||
| 271 | + gp[(fk, ik)] = gp.get((fk, ik), 0) + int(float(gk)) # summed over runs (backfilled + October) | ||||
| 272 | +cols = "llrr" + ("r" if gp else "") | ||||
| 273 | +body = f"\\begin{{tabular}}{{{cols}}}\n\\toprule\nForm & Section (item) & Ledgered sections & Mentions" + (" & Gate passed (calls)" if gp else "") + " \\\\\n\\midrule\n" | ||||
| 274 | +for form, name in (("10-K", "ledger_10k_items"), ("10-Q", "ledger_10q_items")): | ||||
| 275 | + for r in rows(name): | ||||
| 276 | + extra = f" & {n(gp[(form, r['section_name'])])}" if gp and (form, r["section_name"]) in gp else (" & --" if gp else "") | ||||
| 277 | + body += f"{form} & \\texttt{{{r['section_name'].replace('_', '\\_')}}} & {n(r['n'])} & {n(r['mentions'])}{extra} \\\\\n" | ||||
| 278 | + if form == "10-K": body += "\\midrule\n" | ||||
| 93 | body += "\\bottomrule\n\\end{tabular}" | 279 | body += "\\bottomrule\n\\end{tabular}" | ||
| 94 | w("tab_items.tex", body) | 280 | w("tab_items.tex", body) | ||
| 281 | +k10 = {r["section_name"]: r for r in rows("ledger_10k_items")}; q10 = {r["section_name"]: r for r in rows("ledger_10q_items")} | ||||
| 282 | +for key, src in (("tenKmdnaMentions", k10.get("md_and_a")), ("tenQmdnaMentions", q10.get("md_and_a")), ("tenKbusinessMentions", k10.get("business")), | ||||
| 283 | + ("tenQfinancialStatementsMentions", q10.get("financial_statements"))): | ||||
| 284 | + if src: macro(key, n(src["mentions"]), "data/ledger_10k_items.csv / ledger_10q_items.csv", "mentions by item") | ||||
| 285 | +if k10.get("full_body"): macro("tenKfullBodyRows", n(k10["full_body"]["n"]), "data/ledger_10k_items.csv", "n where section_name=full_body") | ||||
| 95 | 286 | ||||
| 96 | −# ---- Table: projects by type ---- | 287 | +# ---- Table 5: projects by type --------------------------------------------------- | ||
| 97 | pt = rows("projects_by_type") | 288 | pt = rows("projects_by_type") | ||
| 98 | −body = "\\begin{tabular}{llrrrrr}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount & Median amount (M USD) & Mentions / project \\\\\n\\midrule\n" | 289 | +ci = {} | ||
| 290 | +ci_path = D / "projects_by_type_ci.csv" | ||||
| 291 | +if ci_path.exists(): | ||||
| 292 | + for r in rows("projects_by_type_ci"): | ||||
| 293 | + lo = next((r[c] for c in r if c.lower() in ("ci_low", "lo", "ci_lo", "median_ci_low")), None) | ||||
| 294 | + hi = next((r[c] for c in r if c.lower() in ("ci_high", "hi", "ci_hi", "median_ci_high")), None) | ||||
| 295 | + if lo not in (None, "") and hi not in (None, ""): | ||||
| 296 | + ci[r["project_type"]] = (float(lo), float(hi)) | ||||
| 297 | +small_n = di("small_n_threshold") or 50 | ||||
| 298 | +# the threshold itself is a documented presentation rule (data/documented_inputs.json), not a measurement | ||||
| 299 | +ad = rows("amount_dist")[0]; q = json.loads(ad["q"]) | ||||
| 300 | +total_median = S.get("amount_total_median_musd", q[2] / 1e6) | ||||
| 301 | +om = {} | ||||
| 302 | +om_path = R / "resolver" / "overmerge_by_type.csv" | ||||
| 303 | +if om_path.exists(): | ||||
| 304 | + om = {r["project_type"]: r for r in rows(om_path.stem, om_path.parent)} | ||||
| 305 | +oc = " & $\\geq 3$ distinct names (\\%)" if om else "" | ||||
| 306 | +body = ("\\begin{tabular}{llrrrrrr" + ("r" if om else "") + "}\n\\toprule\nCode & Category & Projects & Share (\\%) & With amount ($N$) & Median amount (M USD) & " | ||||
| 307 | + "95\\% CI of the median & Mentions / project" + oc + " \\\\\n\\midrule\n") | ||||
| 308 | +pct_sum = 0.0; greyed = [] | ||||
| 99 | for r in pt: | 309 | for r in pt: | ||
| 100 | − body += f"\\texttt{{{r['project_type'].replace('_','\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(r['with_amount'])} & {n(r['median_amount_musd'],1)} & {float(r['mean_mentions']):.2f} \\\\\n" | 310 | + pct_sum += float(r["pct"]); N = int(r["with_amount"]) | ||
| 101 | −body += f"\\midrule\n & Total & {n(S['projects'])} & 100.0 & {n(S['projects_fields'][0])} & 128.5 & {S['mentions']/S['projects']:.2f} \\\\\n\\bottomrule\n\\end{{tabular}}" | 311 | + med = n(r["median_amount_musd"], 1) | ||
| 312 | + if N < small_n: | ||||
| 313 | + med = f"\\textcolor{{gray}}{{{med}}}"; greyed.append(r["project_type"]) | ||||
| 314 | + cic = f"{ci[r['project_type']][0]/1e6:,.1f}--{ci[r['project_type']][1]/1e6:,.1f}".replace(",", "{,}") if r["project_type"] in ci else "--" | ||||
| 315 | + ov = f" & {float(om[r['project_type']]['pct_ge3_names']):.1f}" if om and r["project_type"] in om else (" & --" if om else "") | ||||
| 316 | + body += (f"\\texttt{{{r['project_type'].replace('_', '\\_')}}} & {LAB[r['project_type']]} & {n(r['n'])} & {r['pct']} & {n(N)} & {med} & {cic} & " | ||||
| 317 | + f"{float(r['mean_mentions']):.2f}{ov} \\\\\n") | ||||
| 318 | +body += (f"\\midrule\n & Total & {n(S['projects'])} & {pct_sum:.1f} & {n(S['projects_fields'][0])} & {n(total_median, 1)} & -- & " | ||||
| 319 | + f"{S['mentions']/S['projects']:.2f}" + (" & " if om else "") + f" \\\\\n\\bottomrule\n\\end{{tabular}}") | ||||
| 102 | w("tab_types.tex", body) | 320 | w("tab_types.tex", body) | ||
| 321 | +note = (f"\\item Shares are rounded to one decimal and sum to {pct_sum:.1f}. ``With amount'' counts projects for which at least one mention carries a USD " | ||||
| 322 | + f"figure; ``median amount'' is the median across those projects of the largest amount cited (Section~\\ref{{sec:amounts}} on why amounts are never summed). " | ||||
| 323 | + f"Medians printed in grey rest on fewer than {small_n} projects with an amount ({', '.join('\\texttt{' + g.replace('_', '\\_') + '}' for g in greyed)}) " | ||||
| 324 | + "and should not be quoted without their $N$. " | ||||
| 325 | + + ("The 95\\% interval of the median is a percentile bootstrap (\\texttt{data/projects\\_by\\_type\\_ci.csv}). " if ci | ||||
| 326 | + else "The 95\\% interval column is filled when \\texttt{data/projects\\_by\\_type\\_ci.csv} (bootstrap on the node) is present. ") | ||||
| 327 | + + ("``$\\geq 3$ distinct names'' is the share of projects of the category whose mentions carry three or more distinct project names " | ||||
| 328 | + "(over-merge indicator of the resolution key, \\texttt{results/resolver/overmerge\\_by\\_type.csv}).\n" if om else "\n")) | ||||
| 329 | +w("tab_types_note.tex", note) | ||||
| 330 | +macro("typesPctSum", f"{pct_sum:.1f}", "data/projects_by_type.csv", "sum(pct)") | ||||
| 331 | +macro("amountMedianMusd", n(total_median, 1), "stats.json:amount_total_median_musd or data/amount_dist.csv", "median(total_amount_usd)/1e6") | ||||
| 332 | +macro("smallNThreshold", str(small_n), "data/documented_inputs.json:small_n_threshold", "presentation rule") | ||||
| 333 | +macro("smallNCategories", ", ".join(greyed), "data/projects_by_type.csv", f"with_amount < {small_n}") | ||||
| 334 | +ptd = {r["project_type"]: r for r in pt} | ||||
| 335 | +for code in ("data_center", "ai_initiative", "cloud_migration", "automation", "ma_integration", "capex_program", "partnership", "geographic_expansion", "plant_construction"): | ||||
| 336 | + if code in ptd: | ||||
| 337 | + cam = "".join(p.capitalize() for p in code.split("_")) | ||||
| 338 | + macro(f"share{cam}", ptd[code]["pct"], "data/projects_by_type.csv", f"pct where project_type={code}") | ||||
| 339 | + macro(f"n{cam}", n(ptd[code]["n"]), "data/projects_by_type.csv", f"n where project_type={code}") | ||||
| 340 | + macro(f"withAmount{cam}", n(ptd[code]["with_amount"]), "data/projects_by_type.csv", f"with_amount where project_type={code}") | ||||
| 341 | + macro(f"medianAmount{cam}", n(ptd[code]["median_amount_musd"], 1), "data/projects_by_type.csv", f"median_amount_musd where project_type={code}") | ||||
| 342 | +tech = [c for c in ("data_center", "ai_initiative", "cloud_migration", "automation") if c in ptd] | ||||
| 343 | +macro("techCategoriesCountRange", f"{n(min(int(ptd[c]['n']) for c in tech))}--{n(max(int(ptd[c]['n']) for c in tech))}", "data/projects_by_type.csv", "min/max n over " + ", ".join(tech)) | ||||
| 103 | 344 | ||||
| 104 | −# ---- Table: by sector ---- | 345 | +# ---- Table 6: by sector ------------------------------------------------------------- | ||
| 105 | cs = {r["sector"]: r for r in rows("companies_by_sector")} | 346 | cs = {r["sector"]: r for r in rows("companies_by_sector")} | ||
| 347 | +sec = rows("projects_by_sector") | ||||
| 106 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n" | 348 | body = "\\begin{tabular}{lrrrrr}\n\\toprule\nGICS sector (iShares) & Issuers in universe & of which S\\&P~500 & Issuers with projects & Projects & Projects per issuer \\\\\n\\midrule\n" | ||
| 107 | −for r in rows("projects_by_sector"): | 349 | +for r in sec: | ||
| 108 | c = cs.get(r["sector"], {"n": "", "sp500": ""}) | 350 | c = cs.get(r["sector"], {"n": "", "sp500": ""}) | ||
| 109 | body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" | 351 | body += f"{r['sector']} & {n(c['n']) if c['n'] else ''} & {n(c['sp500']) if c['sp500'] else ''} & {n(r['issuers'])} & {n(r['n'])} & {float(r['per_issuer']):.1f} \\\\\n" | ||
| 110 | body += "\\bottomrule\n\\end{tabular}" | 352 | body += "\\bottomrule\n\\end{tabular}" | ||
| 111 | w("tab_sectors.tex", body) | 353 | w("tab_sectors.tex", body) | ||
| 354 | +secd = {r["sector"]: r for r in sec} | ||||
| 355 | +top4 = sorted(sec, key=lambda r: -float(r["per_issuer"]))[:4] | ||||
| 356 | +vals = [float(r["per_issuer"]) for r in top4] | ||||
| 357 | +macro("sectorTopFourPerIssuerRange", f"{math.floor(min(vals))}--{math.ceil(max(vals))}", "data/projects_by_sector.csv", "floor(min)/ceil(max) of per_issuer over the four highest sectors: " + ", ".join(r["sector"] for r in top4)) | ||||
| 358 | +macro("sectorTopFourNames", ", ".join(r["sector"].lower() for r in top4), "data/projects_by_sector.csv", "four highest per_issuer") | ||||
| 359 | +if "Financials" in secd: macro("sectorFinancialsPerIssuer", f"{round(float(secd['Financials']['per_issuer']))}", "data/projects_by_sector.csv", "round(per_issuer) for Financials") | ||||
| 360 | +for name in ("Health Care", "Industrials"): | ||||
| 361 | + if name in secd: | ||||
| 362 | + cam = name.replace(" ", "") | ||||
| 363 | + macro(f"sectorProjects{cam}", n(secd[name]["n"]), "data/projects_by_sector.csv", f"n for {name}") | ||||
| 364 | + macro(f"sectorIssuers{cam}", n(secd[name]["issuers"]), "data/projects_by_sector.csv", f"issuers for {name}") | ||||
| 112 | 365 | ||||
| 113 | −# ---- Table: status, mention buckets, duration, confidence ---- | 366 | +# ---- Table 7: status, mention buckets, duration ---------------------------------------- | ||
| 114 | st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") | 367 | st = rows("projects_by_status"); mb = rows("mentions_per_project"); du = rows("duration"); cf = rows("confidence_dist") | ||
| 115 | body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" | 368 | body = "\\begin{tabular}{lr@{\\hspace{2em}}lr@{\\hspace{2em}}lr}\n\\toprule\nLatest status & Projects & Mentions per project & Projects & Span of cited years & Projects \\\\\n\\midrule\n" | ||
| 116 | −du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: r["n"] for r in du} | 369 | +du_order = ["0", "1-2", "3-5", ">5", "unknown"]; dus = {r["span_years"]: int(r["n"]) for r in du} | ||
| 117 | rowsn = max(len(st), len(mb), len(du_order)) | 370 | rowsn = max(len(st), len(mb), len(du_order)) | ||
| 118 | for i in range(rowsn): | 371 | for i in range(rowsn): | ||
| 119 | − a = f"\\texttt{{{st[i]['status'].replace('_','\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | 372 | + a = f"\\texttt{{{st[i]['status'].replace('_', '\\_')}}} & {n(st[i]['n'])}" if i < len(st) else "&" | ||
| 120 | b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" | 373 | b = f"{mb[i]['bucket']} & {n(mb[i]['n'])}" if i < len(mb) else "&" | ||
| 121 | − c = f"{du_order[i].replace('>','$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | 374 | + c = f"{du_order[i].replace('>', '$>$')} years & {n(dus[du_order[i]])}" if i < len(du_order) else "&" | ||
| 122 | body += f"{a} & {b} & {c} \\\\\n" | 375 | body += f"{a} & {b} & {c} \\\\\n" | ||
| 123 | body += "\\bottomrule\n\\end{tabular}" | 376 | body += "\\bottomrule\n\\end{tabular}" | ||
| 124 | w("tab_status.tex", body) | 377 | w("tab_status.tex", body) | ||
| 378 | +P = S["projects"]; known = P - dus["unknown"] | ||||
| 379 | +macro("spanKnownN", n(known), "data/duration.csv", "projects - unknown") | ||||
| 380 | +macro("spanUnknownN", n(dus["unknown"]), "data/duration.csv", "n where span_years=unknown") | ||||
| 381 | +macro("spanUnknownShareAll", f"{100*dus['unknown']/P:.1f}", "data/duration.csv", "100*unknown/projects") | ||||
| 382 | +macro("spanZeroShareAll", f"{100*dus['0']/P:.1f}", "data/duration.csv", "100*n(0)/projects") | ||||
| 383 | +macro("spanZeroShareKnown", f"{100*dus['0']/known:.1f}", "data/duration.csv", "100*n(0)/(projects-unknown)") | ||||
| 384 | +macro("spanOverFiveShareAll", f"{100*dus['>5']/P:.1f}", "data/duration.csv", "100*n(>5)/projects") | ||||
| 385 | +macro("spanOverFiveShareKnown", f"{100*dus['>5']/known:.1f}", "data/duration.csv", "100*n(>5)/(projects-unknown)") | ||||
| 386 | +for r in st: | ||||
| 387 | + macro("statusShare" + "".join(p.capitalize() for p in r["status"].split("_")), r["pct"], "data/projects_by_status.csv", f"pct where status={r['status']}") | ||||
| 388 | +mbd = {r["bucket"]: int(r["n"]) for r in mb} | ||||
| 389 | +macro("singleMentionShare", f"{100*mbd['1']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=1)/projects") | ||||
| 390 | +macro("overTenMentionShare", f"{100*mbd['>10']/P:.1f}", "data/mentions_per_project.csv", "100*n(bucket=>10)/projects") | ||||
| 391 | + | ||||
| 392 | +# ---- confidence --------------------------------------------------------------------- | ||||
| 125 | body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n" | 393 | body = "\\begin{tabular}{lr}\n\\toprule\nSelf-reported confidence (rounded) & Mentions \\\\\n\\midrule\n" | ||
| 126 | for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n" | 394 | for r in cf: body += f"{r['conf']} & {n(r['n'])} \\\\\n" | ||
| 127 | body += "\\bottomrule\n\\end{tabular}" | 395 | body += "\\bottomrule\n\\end{tabular}" | ||
| 128 | w("tab_confidence.tex", body) | 396 | w("tab_confidence.tex", body) | ||
| 397 | +M = S["mentions"]; cfd = {float(r["conf"]): int(r["n"]) for r in cf} | ||||
| 398 | +macro("confShareOne", f"{100*cfd.get(1.0, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=1.0)/mentions") | ||||
| 399 | +macro("confShareNine", f"{100*cfd.get(0.9, 0)/M:.1f}", "data/confidence_dist.csv", "100*n(conf=0.9)/mentions") | ||||
| 400 | +macro("confShareLeSix", f"{100*sum(v for k, v in cfd.items() if k <= 0.6)/M:.1f}", "data/confidence_dist.csv", "100*n(conf<=0.6)/mentions") | ||||
| 401 | +if S.get("conf_exact_default_count") is not None: | ||||
| 402 | + macro("confExactDefaultCount", n(S["conf_exact_default_count"]), "stats.json:conf_exact_default_count", "count(round(confidence,3)=0.600)") | ||||
| 129 | 403 | ||||
| 130 | −# ---- Table: amounts ---- | 404 | +# ---- Table 8: amounts ---------------------------------------------------------------- | ||
| 131 | −ad = rows("amount_dist")[0]; q = json.loads(ad["q"]); ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | 405 | +ppi = json.loads(rows("projects_per_issuer")[0]["q"]) | ||
| 132 | body = f"""\\begin{{tabular}}{{lrrrrrr}} | 406 | body = f"""\\begin{{tabular}}{{lrrrrrr}} | ||
| 133 | \\toprule | 407 | \\toprule | ||
| 134 | & P10 & P25 & Median & P75 & P90 & P99 \\\\ | 408 | & P10 & P25 & Median & P75 & P90 & P99 \\\\ | ||
@@ -138,58 +412,296 @@ Projects per issuer ($N$ = {n(S['issuers_with_projects'])}) & {ppi[0]:.0f} & {pp | |||||
| 138 | \\bottomrule | 412 | \\bottomrule | ||
| 139 | \\end{{tabular}}""".replace(",", "{,}") | 413 | \\end{{tabular}}""".replace(",", "{,}") | ||
| 140 | w("tab_amounts.tex", body) | 414 | w("tab_amounts.tex", body) | ||
| 415 | +macro("amountMentions", n(S["mentions_fields"][0]), "stats.json:mentions_fields[0]", "count(amount_usd is not null)") | ||||
| 416 | +macro("amountMentionsShare", f"{100*S['mentions_fields'][0]/M:.1f}", "stats.json", "100*mentions_fields[0]/mentions") | ||||
| 417 | +macro("amountProjects", n(S["projects_fields"][0]), "stats.json:projects_fields[0]", "count(total_amount_usd is not null)") | ||||
| 418 | +macro("amountProjectsShare", f"{100*S['projects_fields'][0]/P:.1f}", "stats.json", "100*projects_fields[0]/projects") | ||||
| 419 | +macro("amountP25Musd", f"{q[1]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[1]/1e6") | ||||
| 420 | +macro("amountP75Musd", f"{q[3]/1e6:,.0f}".replace(",", "{,}"), "data/amount_dist.csv", "q[3]/1e6") | ||||
| 421 | +macro("amountP99Busd", f"{q[5]/1e9:,.1f}", "data/amount_dist.csv", "q[5]/1e9") | ||||
| 422 | +macro("amountSumTusd", f"{float(ad['sum_gusd'])/1e3:,.1f}", "data/amount_dist.csv", "sum_gusd/1e3") | ||||
| 423 | +amt_note = ("\\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values " | ||||
| 424 | + "and estimates and are never summed in the text. ") | ||||
| 425 | +if S.get("amount_max") is not None: | ||||
| 426 | + macro("amountMaxBusd", f"{S['amount_max']/1e9:,.1f}".replace(",", "{,}"), "stats.json:amount_max", "max(total_amount_usd)/1e9") | ||||
| 427 | + amt_note += f"The maximum is {S['amount_max']/1e9:,.1f}~B USD. ".replace(",", "{,}") | ||||
| 428 | +if S.get("amount_ge_50b") is not None: | ||||
| 429 | + macro("amountGeFiftyB", n(S["amount_ge_50b"]), "stats.json:amount_ge_50b", "count(total_amount_usd >= 50e9)") | ||||
| 430 | + amt_note += f"{n(S['amount_ge_50b'])} projects carry an amount of 50~B USD or more. " | ||||
| 431 | +if S.get("amount_le_zero") is not None: | ||||
| 432 | + macro("amountLeZero", n(S["amount_le_zero"]), "stats.json:amount_le_zero", "count(total_amount_usd <= 0)") | ||||
| 433 | +w("tab_amounts_note.tex", amt_note + "\n") | ||||
| 141 | 434 | ||||
| 142 | −# ---- Table: knowledge graph ---- | 435 | +# ---- knowledge graph -------------------------------------------------------------------- | ||
| 143 | kg = rows("kg") | 436 | kg = rows("kg") | ||
| 144 | body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" | 437 | body = "\\begin{tabular}{llr}\n\\toprule\n & Type & Count \\\\\n\\midrule\n" | ||
| 145 | −for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_','\\_')}}} & {n(r['n'])} \\\\\n" | 438 | +for r in kg: body += f"{r['k'].capitalize()} & \\texttt{{{r['node_type'].replace('_', '\\_')}}} & {n(r['n'])} \\\\\n" | ||
| 146 | body += "\\bottomrule\n\\end{tabular}" | 439 | body += "\\bottomrule\n\\end{tabular}" | ||
| 147 | w("tab_graph.tex", body) | 440 | w("tab_graph.tex", body) | ||
| 441 | +macro("kgNodes", n(sum(int(r["n"]) for r in kg if r["k"] == "nodes")), "data/kg.csv", "sum(n) where k=nodes") | ||||
| 442 | +macro("kgEdges", n(sum(int(r["n"]) for r in kg if r["k"] == "edges")), "data/kg.csv", "sum(n) where k=edges") | ||||
| 443 | +for r in kg: | ||||
| 444 | + if r["k"] == "nodes": macro("kgNodes" + r["node_type"].capitalize(), n(r["n"]), "data/kg.csv", f"n where node_type={r['node_type']}") | ||||
| 148 | 445 | ||||
| 149 | −# ---- Table: top issuers ---- | 446 | +# ---- top issuers --------------------------------------------------------------------------- | ||
| 150 | body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" | 447 | body = "\\begin{tabular}{llr}\n\\toprule\nTicker & Issuer & Resolved projects \\\\\n\\midrule\n" | ||
| 151 | −for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&','\\&')} & {n(r['projects'])} \\\\\n" | 448 | +for r in rows("top_issuers")[:12]: body += f"{r['ticker']} & {r['company_name'].title().replace('&', '\\&')} & {n(r['projects'])} \\\\\n" | ||
| 152 | body += "\\bottomrule\n\\end{tabular}" | 449 | body += "\\bottomrule\n\\end{tabular}" | ||
| 153 | w("tab_top_issuers.tex", body) | 450 | w("tab_top_issuers.tex", body) | ||
| 154 | 451 | ||||
| 155 | −# ---- Table: provenance example (Cheniere Corpus Christi) ---- | 452 | +# ---- Table 9: provenance example (Cheniere Corpus Christi) ------------------------------------ | ||
| 156 | ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] | 453 | ex = [r for r in rows("example_provenance") if r["ticker"] == "LNG"] | ||
| 157 | body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n" | 454 | body = "\\begin{tabular}{llllp{6.2cm}}\n\\toprule\nAccession & Filed & Form & Status / amount & Evidence snippet (model-generated description grounded in the section) \\\\\n\\midrule\n" | ||
| 158 | for r in ex[:8]: | 455 | for r in ex[:8]: | ||
| 159 | amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" | 456 | amt = f"{float(r['amount_usd'])/1e9:.1f} B USD" if r["amount_usd"] else "--" | ||
| 160 | − snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:170] | 457 | + snip = r["snippet"].replace("&", "\\&").replace("%", "\\%").replace("$", "\\$")[:DI["snippet_chars"]["value"]] | ||
| 161 | − body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_','\\_')} / {amt} & {snip} \\\\\n" | 458 | + body += f"\\texttt{{{r['accession_number']}}} & {r['filing_date']} & {r['form_type']} & {r['m_status'].replace('_', '\\_')} / {amt} & {snip} \\\\\n" | ||
| 162 | body += "\\bottomrule\n\\end{tabular}" | 459 | body += "\\bottomrule\n\\end{tabular}" | ||
| 163 | w("tab_provenance.tex", body) | 460 | w("tab_provenance.tex", body) | ||
| 164 | −S["example_project"] = {k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")} if ex else {} | 461 | +if ex: | ||
| 462 | + forms = sorted({r["form_type"] for r in ex[:8]}) | ||||
| 463 | + macro("provenanceForms", " and ".join(forms), "data/example_provenance.csv", "distinct form_type of the printed rows") | ||||
| 464 | + macro("provenanceN", str(len(ex[:8])), "data/example_provenance.csv", "rows printed") | ||||
| 465 | + macro("provenanceFirst", ex[0]["filing_date"], "data/example_provenance.csv", "min filing_date") | ||||
| 466 | + macro("provenanceLast", ex[:8][-1]["filing_date"], "data/example_provenance.csv", "max filing_date") | ||||
| 467 | + macro("provenanceMaxAmountBusd", f"{max(float(r['amount_usd']) for r in ex if r['amount_usd'])/1e9:.1f}", "data/example_provenance.csv", "max(amount_usd)/1e9") | ||||
| 468 | + macro("provenanceNameYear", ex[:8][-1]["filing_date"][:4], "data/example_provenance.csv", "filing year of the last mention (representative name source)") | ||||
| 469 | + json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, | ||||
| 470 | + open(DATA_OUT / "example_project.json", "w"), indent=2) | ||||
| 471 | + | ||||
| 472 | +# ---- Table 12: data-model audit (Appendix C) as a longtable ------------------------------------- | ||||
| 473 | +if (D / "datamodel_audit.csv").exists(): | ||||
| 474 | + body = ("\\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}}\n\\caption{Tables required by the research design, current implementation and status}" | ||||
| 475 | + "\\label{tab:datamodel}\\\\\n\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endfirsthead\n" | ||||
| 476 | + "\\toprule\nRequired table & Current implementation (database, table, key fields) & Status \\\\\n\\midrule\n\\endhead\n\\bottomrule\n\\endfoot\n") | ||||
| 477 | + for r in rows("datamodel_audit"): | ||||
| 478 | + body += f"\\texttt{{{r['required_table'].replace('_', '\\_')}}} & {r['implementation']} & {r['status']} \\\\\n" | ||||
| 479 | + body += "\\end{longtable}" | ||||
| 480 | + w("tab_datamodel.tex", body) | ||||
| 481 | + | ||||
| 482 | +# ---- Appendix B: candidate gate regex and system prompt, read from the frozen snapshot ---------- | ||||
| 483 | +def _snapshot(name): | ||||
| 484 | + for base in (SNAP, ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" / "project-db" / "spid"): | ||||
| 485 | + for p in (base / name, base / "extract" / name): | ||||
| 486 | + if p.exists(): return p | ||||
| 487 | + return None | ||||
| 488 | + | ||||
| 489 | +rules_py = _snapshot("rules.py") | ||||
| 490 | +if rules_py: | ||||
| 491 | + tree = ast.parse(rules_py.read_text()) | ||||
| 492 | + pieces, flags = None, "" | ||||
| 493 | + for node in ast.walk(tree): | ||||
| 494 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_PROJECT_GATE" for t in node.targets): | ||||
| 495 | + call = node.value; arg = call.args[0] | ||||
| 496 | + # the pattern is written as adjacent raw-string literals; keep the author's line breaks | ||||
| 497 | + seg = rules_py.read_text().splitlines()[arg.lineno - 1: arg.end_lineno] | ||||
| 498 | + pieces = [re.match(r'\s*r"(.*)"\s*,?\s*$', s).group(1) for s in seg] | ||||
| 499 | + flags = ", ".join(ast.unparse(a) for a in call.args[1:]) | ||||
| 500 | + if pieces: | ||||
| 501 | + full = "".join(pieces) | ||||
| 502 | + lines = "\n".join(pieces) | ||||
| 503 | + w("gate_regex.tex", "\\begin{verbatim}\n" + lines + "\n\\end{verbatim}\n") | ||||
| 504 | + macro("gateRegexFlags", esc(flags), str(rules_py.relative_to(ROOT)) if rules_py.is_relative_to(ROOT) else str(rules_py), "re.compile flags") | ||||
| 505 | + macro("gateRegexLength", str(len(full)), "rules.py:_PROJECT_GATE", "len(pattern)") | ||||
| 506 | + macro("gateRegexAlternatives", str(full.count("|") + 1), "rules.py:_PROJECT_GATE", "number of alternatives") | ||||
| 507 | + | ||||
| 508 | +llm_py, tax_py = _snapshot("llm.py"), _snapshot("taxonomy.py") | ||||
| 509 | +if llm_py and tax_py: | ||||
| 510 | + ttree = ast.parse(tax_py.read_text()); types = None | ||||
| 511 | + for node in ast.walk(ttree): | ||||
| 512 | + if isinstance(node, (ast.Assign, ast.AnnAssign)): | ||||
| 513 | + tg = node.targets[0] if isinstance(node, ast.Assign) else node.target | ||||
| 514 | + if isinstance(tg, ast.Name) and tg.id == "PROJECT_TYPES": | ||||
| 515 | + types = ast.literal_eval(node.value) | ||||
| 516 | + ltree = ast.parse(llm_py.read_text()); tmpl = None | ||||
| 517 | + for node in ast.walk(ltree): | ||||
| 518 | + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "_SYSTEM" for t in node.targets): | ||||
| 519 | + v = node.value | ||||
| 520 | + if isinstance(v, ast.BinOp) and isinstance(v.left, ast.Constant): tmpl = v.left.value | ||||
| 521 | + elif isinstance(v, ast.Constant): tmpl = v.value | ||||
| 522 | + if types and tmpl and "%s" in tmpl: | ||||
| 523 | + labels = "\n".join(f" - {k}: {v['label']}" for k, v in types.items()) | ||||
| 524 | + prompt = tmpl % labels | ||||
| 525 | + w("prompt_verbatim.tex", "\\begin{verbatim}\n" + prompt + "\n\\end{verbatim}\n") | ||||
| 526 | + macro("taxonomyN", str(len(types)), "taxonomy.py:PROJECT_TYPES", "len") | ||||
| 527 | + body = "\\begin{tabular}{ll}\n\\toprule\nCode & Label \\\\\n\\midrule\n" | ||||
| 528 | + for k, v in types.items(): body += f"\\texttt{{{k.replace('_', '\\_')}}} & {esc(v['label'])} \\\\\n" | ||||
| 529 | + body += "\\bottomrule\n\\end{tabular}" | ||||
| 530 | + w("tab_taxonomy.tex", body) | ||||
| 165 | 531 | ||||
| 166 | −# ---- Figures ---- | 532 | +# ---- Stage D audit tables from results/ (generic CSV -> tabular; skipped when absent) --------------- | ||
| 533 | +def csv_table(texname, path, caption_cols=None, max_rows=60): | ||||
| 534 | + if not path.exists(): | ||||
| 535 | + print("skip", texname, "(missing", path.relative_to(ROOT), ")"); return | ||||
| 536 | + with open(path, newline="") as f: | ||||
| 537 | + rd = csv.DictReader(f); cols = rd.fieldnames; data = list(rd) | ||||
| 538 | + def fmt(v): | ||||
| 539 | + try: | ||||
| 540 | + x = float(v) | ||||
| 541 | + if x.is_integer() and abs(x) >= 1e3: return n(x) | ||||
| 542 | + return v if x.is_integer() else f"{x:,.3f}".rstrip("0").rstrip(".").replace(",", "{,}") | ||||
| 543 | + except ValueError: | ||||
| 544 | + return esc(v) | ||||
| 545 | + spec = "l" + "r" * (len(cols) - 1) | ||||
| 546 | + body = f"\\begin{{tabular}}{{{spec}}}\n\\toprule\n" + " & ".join(esc(c) for c in cols) + " \\\\\n\\midrule\n" | ||||
| 547 | + for r in data[:max_rows]: body += " & ".join(fmt(r[c]) for c in cols) + " \\\\\n" | ||||
| 548 | + if len(data) > max_rows: body += f"\\multicolumn{{{len(cols)}}}{{l}}{{\\ldots {len(data) - max_rows} more rows in \\texttt{{{esc(str(path.relative_to(ROOT)))}}}}} \\\\\n" | ||||
| 549 | + body += "\\bottomrule\n\\end{tabular}" | ||||
| 550 | + w(texname, body) | ||||
| 551 | + | ||||
| 552 | +csv_table("tab_spec_curve.tex", R / "resolver" / "spec_curve.csv") | ||||
| 553 | +csv_table("tab_grounding.tex", R / "grounding" / "grounding_by_field_form_item.csv") | ||||
| 554 | +csv_table("tab_focus.tex", R / "focus" / "focus_coverage_by_form_item.csv") | ||||
| 555 | +csv_table("tab_cohort.tex", R / "runs" / "cohort_by_run.csv" if (R / "runs" / "cohort_by_run.csv").exists() else D / "cohort_by_run.csv") | ||||
| 556 | +csv_table("tab_fp_classes.tex", R / "fp_classes" / "financing_regex_by_item_type.csv") | ||||
| 557 | +csv_table("tab_timeseries_cohort.tex", R / "timeseries" / "first_seen_fixed_cohort.csv") | ||||
| 558 | +csv_table("tab_benchmark_8k.tex", R / "benchmark_8k" / "item_vs_type.csv") | ||||
| 559 | +csv_table("tab_rules_agreement.tex", R / "rules" / "agreement_by_form_item.csv") | ||||
| 560 | + | ||||
| 561 | +# ---- macros from the Stage D audit results (results/**.json; skipped when absent) ------------------ | ||||
| 562 | +def jload(rel): | ||||
| 563 | + p = R / rel | ||||
| 564 | + return json.load(open(p)) if p.exists() else {} | ||||
| 565 | + | ||||
| 566 | +def jmacro(name, obj, key, src, fmt=None): | ||||
| 567 | + v = obj.get(key) if isinstance(obj, dict) else None | ||||
| 568 | + if v is None or isinstance(v, (list, dict)): return | ||||
| 569 | + macro(name, fmt(v) if fmt else (n(v) if isinstance(v, (int, float)) and float(v).is_integer() else str(v)), src, key) | ||||
| 570 | + | ||||
| 571 | +gt = jload("ledger/gate_pass_totals.json") | ||||
| 572 | +for k, mname, fmt in (("calls", "callsTotal", None), ("gate_pass", "gatePassSections", None), ("gate_rejected_zero", "gateRejectedSections", None), | ||||
| 573 | + ("lt30_words", "ledgerUnderThirtyWords", None), ("productive_pct_calls", "productiveShareCalls", lambda v: f"{v:.1f}"), | ||||
| 574 | + ("mentions_per_call", "mentionsPerCall", lambda v: f"{v:.2f}"), ("usd_per_call_paper_total_775", "usdPerCall", lambda v: f"{v:.4f}"), | ||||
| 575 | + ("october_calls", "octoberCalls", None), ("october_rows", "octoberRows", None)): | ||||
| 576 | + jmacro(mname, gt, k, "results/ledger/gate_pass_totals.json", fmt) | ||||
| 577 | +tp = jload("ledger/throughput.json") | ||||
| 578 | +for k, mname, fmt in (("wall_clock_hours_first_to_last_flush", "octoberWallClockHours", lambda v: f"{v:.1f}"), ("largest_gap_hours", "outageHours", lambda v: f"{v:.1f}"), | ||||
| 579 | + ("effective_hours_excluding_largest_gap", "octoberEffectiveHours", lambda v: f"{v:.1f}"), | ||||
| 580 | + ("rows_per_second_effective", "octoberRowsPerSecondEffective", lambda v: f"{v:.1f}"), ("rows_per_second_wall_clock", "octoberRowsPerSecondWallClock", lambda v: f"{v:.1f}"), | ||||
| 581 | + ("calls_per_second_effective", "octoberCallsPerSecondEffective", lambda v: f"{v:.1f}"), ("october_first_flush", "octoberFirstFlush", lambda v: str(v)[:16]), | ||||
| 582 | + ("october_last_flush", "octoberLastFlush", lambda v: str(v)[:16])): | ||||
| 583 | + jmacro(mname, tp, k, "results/ledger/throughput.json", fmt) | ||||
| 584 | +bb = jload("ledger/backfill_block.json") | ||||
| 585 | +for k, mname in (("rows_at_min_timestamp", "backfilledRows"), ("june_log_sections", "juneLogSections"), ("block_lt30_words", "backfilledUnderThirtyWords"), | ||||
| 586 | + ("block_stored_mentions", "backfilledMentions"), ("block_ciks", "backfilledCiks"), ("block_gate_fail", "backfilledGateFail"), ("rows_in_outage_window", "rowsInOutageWindow")): | ||||
| 587 | + jmacro(mname, bb, k, "results/ledger/backfill_block.json") | ||||
| 588 | +if isinstance(bb.get("outage_window"), list) and len(bb["outage_window"]) == 2: | ||||
| 589 | + macro("outageWindowStart", str(bb["outage_window"][0])[:16], "results/ledger/backfill_block.json", "outage_window[0]") | ||||
| 590 | + macro("outageWindowEnd", str(bb["outage_window"][1])[:16], "results/ledger/backfill_block.json", "outage_window[1]") | ||||
| 591 | +ls_ = jload("ledger/log_summary.json") | ||||
| 592 | +oc_ = ls_.get("october", {}) if isinstance(ls_.get("october"), dict) else {} | ||||
| 593 | +for k, mname in (("llm_call_failed_total", "llmCallFailedTotal"), ("llm_call_failed_inside_window", "llmCallFailedInsideOutage"), ("llm_call_failed_outside_window", "llmCallFailedOutsideOutage")): | ||||
| 594 | + jmacro(mname, oc_, k, "results/ledger/log_summary.json:october") | ||||
| 595 | +if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | ||||
| 596 | + for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) | ||||
| 597 | +jn_ = ls_.get("june", {}) if isinstance(ls_.get("june"), dict) else {} | ||||
| 598 | +jmacro("juneLlmCallFailed", jn_, "llm_call_failed_total", "results/ledger/log_summary.json:june") | ||||
| 599 | +co = jload("ledger/collisions.json") | ||||
| 600 | +for k, mname in (("sections_ledger_gt_stored", "collisionSections"), ("excess_mentions", "collisionMentions")): | ||||
| 601 | + jmacro(mname, co, k, "results/ledger/collisions.json") | ||||
| 602 | +cp = jload("resolver/composition.json") | ||||
| 603 | +for k, mname in (("projects_keyed_general", "projectsKeyedGeneral"), ("mentions_keyed_general", "mentionsKeyedGeneral")): | ||||
| 604 | + jmacro(mname, cp, k, "results/resolver/composition.json") | ||||
| 605 | +if isinstance(cp.get("projects_by_key_branch"), list): | ||||
| 606 | + for br, v in cp["projects_by_key_branch"]: | ||||
| 607 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")), n(v), "results/resolver/composition.json", f"projects_by_key_branch[{br}]") | ||||
| 608 | + macro("projectsKeyed" + "".join(p.capitalize() for p in str(br).split("_")) + "Pct", f"{100*v/P:.1f}", "results/resolver/composition.json", f"100*projects_by_key_branch[{br}]/projects") | ||||
| 609 | +ti = jload("resolver/ties_idempotence.json") | ||||
| 610 | +for k, mname in (("projects_with_tie_at_max_confidence", "projectsTieMaxConfidence"), ("projects_with_tie_and_distinct_names", "projectsTieDistinctNames"), | ||||
| 611 | + ("projects_with_status_tie_on_last_seen", "projectsStatusTie"), ("mentions_at_confidence_1", "mentionsAtConfidenceOne")): | ||||
| 612 | + jmacro(mname, ti, k, "results/resolver/ties_idempotence.json") | ||||
| 613 | +sc_path = R / "resolver" / "spec_curve.csv" | ||||
| 614 | +if sc_path.exists(): | ||||
| 615 | + sc = rows(sc_path.stem, sc_path.parent) | ||||
| 616 | + thr = min(sc, key=lambda r: float(r["threshold"]))["threshold"] | ||||
| 617 | + at = [r for r in sc if r["threshold"] == thr] | ||||
| 618 | + pr = [int(float(r["projects"])) for r in at] | ||||
| 619 | + macro("projectCountRangeKeys", f"{n(min(pr))}--{n(max(pr))}", "results/resolver/spec_curve.csv", f"min/max projects over keys at threshold {thr}") | ||||
| 620 | + macro("projectCountRangeKeysN", str(len(at)), "results/resolver/spec_curve.csv", f"number of keys at threshold {thr}") | ||||
| 621 | + pall = [int(float(r["projects"])) for r in sc] | ||||
| 622 | + macro("projectCountRangeAll", f"{n(min(pall))}--{n(max(pall))}", "results/resolver/spec_curve.csv", "min/max projects over all keys x thresholds") | ||||
| 623 | + cur = [r for r in at if "current" in r["key"].lower()] | ||||
| 624 | + if cur: macro("projectCountCurrentKey", n(cur[0]["projects"]), "results/resolver/spec_curve.csv", f"key={cur[0]['key']} at {thr}") | ||||
| 625 | +tn_path = R / "resolver" / "top_name_tokens.csv" | ||||
| 626 | +if tn_path.exists(): | ||||
| 627 | + tn = rows(tn_path.stem, tn_path.parent)[:10] | ||||
| 628 | + macro("topNameTokens", ", ".join(f"\\texttt{{{esc(r['name_tok'])}}} ({float(r['pct_of_tokenkeyed_projects']):.1f}\\%)" for r in tn), "results/resolver/top_name_tokens.csv", "top-10 tokens, pct of token-keyed projects") | ||||
| 629 | + macro("topNameTokensProjectsPct", f"{sum(float(r['pct_of_tokenkeyed_projects']) for r in tn):.1f}", "results/resolver/top_name_tokens.csv", "sum of pct_of_tokenkeyed_projects over the top 10") | ||||
| 630 | +csv_table("tab_top_name_tokens.tex", R / "resolver" / "top_name_tokens.csv", max_rows=20) | ||||
| 631 | +csv_table("tab_overmerge.tex", R / "resolver" / "overmerge_by_type.csv") | ||||
| 632 | +csv_table("tab_june_october.tex", R / "resolver" / "june_october_reconciliation.csv") | ||||
| 633 | +csv_table("tab_table3_calls.tex", R / "ledger" / "table3_ledger_vs_stored.csv") | ||||
| 634 | + | ||||
| 635 | +# ---- headline macros --------------------------------------------------------------------------------- | ||||
| 636 | +macro("projects", n(P), "stats.json:projects", "count(projects)") | ||||
| 637 | +macro("mentions", n(M), "stats.json:mentions", "count(project_mentions)") | ||||
| 638 | +macro("issuersWithProjects", n(S["issuers_with_projects"]), "stats.json:issuers_with_projects", "count(distinct cik)") | ||||
| 639 | +macro("issuersWithFilings", n(S["issuers_with_filings"]), "stats.json:issuers_with_filings", "count(distinct cik) in filings") | ||||
| 640 | +macro("sectionsTotal", n(S["sections_total"]), "stats.json:sections_total", "count(filing_sections)") | ||||
| 641 | +macro("chunks", n(S["chunks"][0]), "stats.json:chunks[0]", "count(filing_chunks)") | ||||
| 642 | +macro("projectsWithLocation", n(S["projects_fields"][1]), "stats.json:projects_fields[1]", "count(canonical_location is not null)") | ||||
| 643 | +macro("projectsTokenKeyedShare", f"{100*(P - S['projects_fields'][1])/P:.1f}", "stats.json:projects_fields", "100*(projects - with location)/projects") | ||||
| 644 | +macro("projectsPerIssuerMean", f"{float(rows('projects_per_issuer')[0]['mean']):.1f}", "data/projects_per_issuer.csv", "mean") | ||||
| 645 | +macro("projectsPerIssuerMedian", f"{ppi[2]:.0f}", "data/projects_per_issuer.csv", "q[2]") | ||||
| 646 | +macro("extractedAtFirst", S["extracted_at_range"][0][:19], "stats.json:extracted_at_range[0]", "min(extracted_at)") | ||||
| 647 | +macro("extractedAtLast", S["extracted_at_range"][1][:19], "stats.json:extracted_at_range[1]", "max(extracted_at)") | ||||
| 648 | +for key, mname in (("ledger_10k_mdna_median_words", "tenKmdnaMedianWords"), ("annual_panel_firms", "annualPanelFirms"), ("sp500_ciks", "spFiveHundredCiks"), | ||||
| 649 | + ("backfilled_rows", "backfilledRows"), ("backfilled_mentions", "backfilledMentions"), ("outage_rows", "outageRows"), | ||||
| 650 | + ("llm_failed_outside_outage", "llmFailedOutsideOutage"), ("llm_failed_inside_outage", "llmFailedInsideOutage"), ("june_failed_calls", "juneFailedCalls")): | ||||
| 651 | + if S.get(key) is not None: macro(mname, n(S[key]), f"stats.json:{key}", "export_stats.py") | ||||
| 652 | +if S.get("spid_runs"): | ||||
| 653 | + for k, v in S["spid_runs"].items(): | ||||
| 654 | + if isinstance(v, dict): | ||||
| 655 | + for kk, vv in v.items(): | ||||
| 656 | + if vv is not None: macro("run" + "".join(p.capitalize() for p in k.split("_")) + "".join(p.capitalize() for p in kk.split("_")), esc(vv), f"stats.json:spid_runs.{k}.{kk}", "export_stats.py") | ||||
| 657 | +if di("brief_definition"): macro("brief", di("brief_definition"), "data/documented_inputs.json:brief_definition", "documented input (author to confirm)") | ||||
| 658 | + | ||||
| 659 | +# ---- Figures --------------------------------------------------------------------------------------------- | ||||
| 167 | BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a" | 660 | BLUE, GOLD, GREY = "#16365c", "#c9a227", "#8a8a8a" | ||
| 168 | yrs = [int(r["yr"]) for r in iy] | 661 | yrs = [int(r["yr"]) for r in iy] | ||
| 169 | fig, ax1 = plt.subplots(figsize=(7.2, 3.4)) | 662 | fig, ax1 = plt.subplots(figsize=(7.2, 3.4)) | ||
| 170 | ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)") | 663 | ax1.bar(yrs, [int(r["filings"]) for r in iy], color=BLUE, alpha=.85, label="Filings (10-K, 10-Q, 8-K)") | ||
| 171 | ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year") | 664 | ax1.set_ylabel("Filings per year"); ax1.set_xlabel("Filing year") | ||
| 665 | +ax1.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||||
| 172 | ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing") | 666 | ax2 = ax1.twinx(); ax2.plot(yrs, [int(r["issuers"]) for r in iy], color=GOLD, marker="o", lw=2, label="Issuers with $\\geq$1 filing") | ||
| 173 | −ax2.set_ylabel("Issuers"); ax2.set_ylim(0, 2700) | 667 | +ax2.set_ylabel("Issuers"); ax2.set_ylim(0, U["n"] * 1.05) | ||
| 174 | h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8) | 668 | h1, l1 = ax1.get_legend_handles_labels(); h2, l2 = ax2.get_legend_handles_labels(); ax1.legend(h1 + h2, l1 + l2, loc="upper left", frameon=False, fontsize=8) | ||
| 175 | for a in (ax1, ax2): a.spines[["top"]].set_visible(False) | 669 | for a in (ax1, ax2): a.spines[["top"]].set_visible(False) | ||
| 176 | −fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf"); plt.close(fig) | 670 | +fig.tight_layout(); fig.savefig(F / "fig_coverage.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||
| 177 | 671 | ||||
| 178 | fig, ax = plt.subplots(figsize=(7.2, 5.2)) | 672 | fig, ax = plt.subplots(figsize=(7.2, 5.2)) | ||
| 179 | labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] | 673 | labs = [LAB[r["project_type"]].replace("\\&", "&") for r in pt][::-1]; vals = [int(r["n"]) for r in pt][::-1] | ||
| 180 | ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False) | 674 | ax.barh(labs, vals, color=BLUE); ax.set_xlabel("Resolved projects"); ax.spines[["top", "right"]].set_visible(False) | ||
| 181 | −for i, v in enumerate(vals): ax.text(v + 300, i, f"{v:,}", va="center", fontsize=7) | 675 | +for i, v in enumerate(vals): ax.text(v + max(vals) * 0.007, i, f"{v:,}", va="center", fontsize=7) | ||
| 182 | −fig.tight_layout(); fig.savefig(F / "fig_types.pdf"); plt.close(fig) | 676 | +ax.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||
| 677 | +fig.tight_layout(); fig.savefig(F / "fig_types.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||||
| 183 | 678 | ||||
| 184 | −fig, ax = plt.subplots(figsize=(7.2, 3.2)) | 679 | +fig, (ax, axb) = plt.subplots(2, 1, figsize=(7.2, 5.6), sharex=True, gridspec_kw={"height_ratios": [3, 2]}) | ||
| 185 | ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions") | 680 | ax.plot(yrs, [int(my[str(y)]["mentions"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Project mentions") | ||
| 186 | ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed") | 681 | ax.plot(yrs, [int(py[str(y)]["n"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed") | ||
| 187 | −ax.set_xlabel("Filing year"); ax.set_ylabel("Count"); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | 682 | +ax.set_ylabel("Count"); ax.set_ylim(0, None); ax.legend(frameon=False, fontsize=8); ax.spines[["top", "right"]].set_visible(False) | ||
| 188 | −fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf"); plt.close(fig) | 683 | +ax.axvspan(yrs[0] - 0.5, yrs[0] + 0.5, color=GREY, alpha=.15); ax.axvspan(yrs[-1] - 0.5, yrs[-1] + 0.5, color=GREY, alpha=.15) | ||
| 684 | +ax.text(yrs[0], ax.get_ylim()[1] * 0.97, "left-censored", fontsize=7, ha="center", va="top", color=GREY) | ||||
| 685 | +ax.text(yrs[-1], ax.get_ylim()[1] * 0.97, "partial year", fontsize=7, ha="center", va="top", color=GREY) | ||||
| 686 | +axb.plot(yrs, [int(my[str(y)]["mentions"]) / int(my[str(y)]["issuers"]) for y in yrs], color=BLUE, marker="o", lw=2, label="Mentions per issuer with $\\geq$1 mention") | ||||
| 687 | +axb.plot(yrs, [int(py[str(y)]["n"]) / int(my[str(y)]["issuers"]) for y in yrs], color=GOLD, marker="s", lw=2, label="Projects first observed per issuer with $\\geq$1 mention") | ||||
| 688 | +axb.set_xlabel("Filing year"); axb.set_ylabel("Per issuer"); axb.set_ylim(0, None); axb.legend(frameon=False, fontsize=8); axb.spines[["top", "right"]].set_visible(False) | ||||
| 689 | +axb.xaxis.set_major_locator(MaxNLocator(integer=True)) | ||||
| 690 | +fig.tight_layout(); fig.savefig(F / "fig_mentions_year.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||||
| 691 | +macro("newProjectsPerIssuerFirstYear", f"{int(py[str(y_first)]['n'])/int(my[str(y_first)]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | ||||
| 692 | +macro("newProjectsPerIssuerLastFullYear", f"{int(py[str(inner[-1])]['n'])/int(my[str(inner[-1])]['issuers']):.1f}", "data/projects_by_first_seen_year.csv, mentions_by_year.csv", "first seen / issuers with mentions") | ||||
| 189 | 693 | ||||
| 190 | fig, ax = plt.subplots(figsize=(7.2, 3.0)) | 694 | fig, ax = plt.subplots(figsize=(7.2, 3.0)) | ||
| 191 | −ax.bar([r["conf"] for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | 695 | +ax.bar([float(r["conf"]) for r in cf], [int(r["n"]) for r in cf], width=0.08, color=BLUE) | ||
| 192 | ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False) | 696 | ax.set_xlabel("Self-reported extraction confidence"); ax.set_ylabel("Mentions"); ax.spines[["top", "right"]].set_visible(False) | ||
| 193 | −fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf"); plt.close(fig) | 697 | +fig.tight_layout(); fig.savefig(F / "fig_confidence.pdf", metadata={"CreationDate": None}); plt.close(fig) | ||
| 194 | print("figures ok") | 698 | print("figures ok") | ||
| 195 | −json.dump(S, open(D / "stats.json", "w"), indent=2, default=str) | 699 | + | ||
| 700 | +# ---- inline-number macros --------------------------------------------------------------------------------- | ||||
| 701 | +lines = ["% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \\SPID<name> in the text.", | ||||
| 702 | + "% Source and formula of every macro: results/inline_numbers.json"] | ||||
| 703 | +for k, v in INLINE.items(): | ||||
| 704 | + lines.append(f"\\newcommand{{\\SPID{k}}}{{{v['value']}}}") | ||||
| 705 | +(T / "numbers.tex").write_text("\n".join(lines) + "\n") | ||||
| 706 | +json.dump(INLINE, open(RES_OUT / "inline_numbers.json", "w"), indent=1, ensure_ascii=False) | ||||
| 707 | +print(f"numbers.tex: {len(INLINE)} macros; stats.json untouched") | ||||
added
data/datamodel_audit.csv
+12 −0
@@ -0,0 +1,12 @@ | |||||
| 1 | +required_table,implementation,status | ||||
| 2 | +universe_snapshots,"\texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded","Partial: no file hash, no membership dates, no match confidence" | ||||
| 3 | +issuers,\texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only),"Partial: no name history, no parent/subsidiary links" | ||||
| 4 | +filings,"\texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status)","Implemented; no acceptance timestamp, no checksum" | ||||
| 5 | +sections,"\texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts)",Implemented; no character offsets into raw HTML | ||||
| 6 | +project_mentions,"\texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context)","Implemented; no evidence span, no prompt hash, no five-way label" | ||||
| 7 | +projects,"\texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years)",Implemented (deterministic key) | ||||
| 8 | +project_mention_links,implicit in the key (every mention maps to exactly one project),"Proposed: explicit table with score, resolver version, adjudication" | ||||
| 9 | +project_events,"\texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet)","Partial: mention rows, not typed events" | ||||
| 10 | +project_amounts,\texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects,"Proposed: type, currency, scale, basis, realization flag" | ||||
| 11 | +extraction_runs,"\texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs","Partial: no prompt/schema/taxonomy hashes, no call status" | ||||
| 12 | +human_labels,---,Proposed | ||||
added
data/documented_inputs.json
+23 −0
@@ -0,0 +1,23 @@ | |||||
| 1 | +{ | ||||
| 2 | + "_about": "Values that no query over the exported data can reproduce. Each carries its source. Read by code/make_tables.py; never used as a measurement.", | ||||
| 3 | + "iwv_equity_lines_reported": { | ||||
| 4 | + "value": 2587, | ||||
| 5 | + "source": "count reported by the build log of scripts/build_universe.py (2026-10-27); the iShares latest-holdings.csv used by the build is not archived (verified on M3U96b, Stage D feasibility report); not reconstructible", | ||||
| 6 | + "status": "not reconstructible from archived artefacts" | ||||
| 7 | + }, | ||||
| 8 | + "small_n_threshold": { | ||||
| 9 | + "value": 50, | ||||
| 10 | + "source": "presentation rule adopted in the Stage D plan (L1): medians resting on fewer than 50 projects with an amount are printed in grey with their N", | ||||
| 11 | + "status": "convention" | ||||
| 12 | + }, | ||||
| 13 | + "brief_definition": { | ||||
| 14 | + "value": "the research-design brief, i.e. the specification of required tables, keys and validation gates against which \\texttt{docs/implementation\\_audit.md} audits the build", | ||||
| 15 | + "source": "usage in docs/implementation_audit.md and paper/sections/resolution.tex; wording to be confirmed by the authors (Stage E)", | ||||
| 16 | + "status": "author to confirm" | ||||
| 17 | + }, | ||||
| 18 | + "snippet_chars": { | ||||
| 19 | + "value": 170, | ||||
| 20 | + "source": "presentation rule of Table 9 (first characters of the stored snippet shown)", | ||||
| 21 | + "status": "convention" | ||||
| 22 | + } | ||||
| 23 | +} | ||||
| \ No newline at end of file | |||||
added
data/example_project.json
+11 −0
@@ -0,0 +1,11 @@ | |||||
| 1 | +{ | ||||
| 2 | + "project_id": "3e2705ba5b2dd27408b25f0145e34250", | ||||
| 3 | + "ticker": "LNG", | ||||
| 4 | + "project_name": "Corpus Christi Liquefaction Stage 3 Project", | ||||
| 5 | + "project_type": "plant_construction", | ||||
| 6 | + "status": "in_progress", | ||||
| 7 | + "total_amount_usd": "11500000000.0", | ||||
| 8 | + "n_mentions": "8", | ||||
| 9 | + "first_seen": "2014-04-08", | ||||
| 10 | + "last_seen": "2022-03-07" | ||||
| 11 | +} | ||||
| \ No newline at end of file | |||||
modified
paper/figures/fig_confidence.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_coverage.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_mentions_year.pdf
+0 −0
Binary file not shown.
modified
paper/figures/fig_types.pdf
+0 −0
Binary file not shown.
modified
paper/main.tex
+2 −0
@@ -91,6 +91,8 @@ | |||||
| 91 | \usetikzlibrary{positioning,arrows.meta,shapes.misc} | 91 | \usetikzlibrary{positioning,arrows.meta,shapes.misc} | ||
| 92 | \usepackage{pdflscape} | 92 | \usepackage{pdflscape} | ||
| 93 | \usepackage{adjustbox} | 93 | \usepackage{adjustbox} | ||
| 94 | +\usepackage{upquote} % straight quotes in verbatim (extraction contract, gate regex) | ||||
| 95 | +\input{tables/numbers} % generated inline-number macros (code/make_tables.py); never edit by hand | ||||
| 94 | 96 | ||||
| 95 | % ============================================================================ | 97 | % ============================================================================ | ||
| 96 | % METADATA — Edit these fields for each working paper | 98 | % METADATA — Edit these fields for each working paper | ||
added
paper/tables/gate_regex.tex
+5 −0
@@ -0,0 +1,5 @@ | |||||
| 1 | +\begin{verbatim} | ||||
| 2 | +\b(project|initiative|program|invest|expan|construct|build|launch|develop| | ||||
| 3 | +deploy|implement|acqui|partner|facilit|capacity|capital|migrat|transform| | ||||
| 4 | +pipeline|plant|data ?cent|automat|sustainab|renewable|restructur)\w* | ||||
| 5 | +\end{verbatim} | ||||
added
paper/tables/numbers.tex
+209 −0
@@ -0,0 +1,209 @@ | |||||
| 1 | +% Generated by code/make_tables.py — inline numbers of the paper. Do not edit; use \SPID<name> in the text. | ||||
| 2 | +% Source and formula of every macro: results/inline_numbers.json | ||||
| 3 | +\newcommand{\SPIDiwvEquityLines}{2{,}587} | ||||
| 4 | +\newcommand{\SPIDuniverseN}{2{,}564} | ||||
| 5 | +\newcommand{\SPIDrussellDistinctCiks}{2{,}555} | ||||
| 6 | +\newcommand{\SPIDrussellMappedLines}{2{,}573} | ||||
| 7 | +\newcommand{\SPIDrussellUnmatched}{7} | ||||
| 8 | +\newcommand{\SPIDrussellIndexConstituents}{2{,}996} | ||||
| 9 | +\newcommand{\SPIDrussellProxyCoveragePct}{85.3} | ||||
| 10 | +\newcommand{\SPIDiwvHoldingsCount}{2{,}588} | ||||
| 11 | +\newcommand{\SPIDiwvReplicationMethod}{representative sampling} | ||||
| 12 | +\newcommand{\SPIDfilingDateFirst}{2010-01-04} | ||||
| 13 | +\newcommand{\SPIDfilingDateLast}{2026-10-27} | ||||
| 14 | +\newcommand{\SPIDfilingsTotal}{527{,}548} | ||||
| 15 | +\newcommand{\SPIDfilingsNotParsed}{16} | ||||
| 16 | +\newcommand{\SPIDfilingsMissingOnDisk}{2} | ||||
| 17 | +\newcommand{\SPIDrawGb}{254.5} | ||||
| 18 | +\newcommand{\SPIDfirstSeenYearMin}{2011} | ||||
| 19 | +\newcommand{\SPIDfirstSeenYearMax}{2025} | ||||
| 20 | +\newcommand{\SPIDfirstSeenRangeInner}{6{,}240--8{,}216} | ||||
| 21 | +\newcommand{\SPIDfirstSeenFirstYear}{7{,}835} | ||||
| 22 | +\newcommand{\SPIDfirstSeenInnerMean}{7{,}105} | ||||
| 23 | +\newcommand{\SPIDmentionsFirstYear}{11{,}908} | ||||
| 24 | +\newcommand{\SPIDmentionsLastFullYear}{23{,}220} | ||||
| 25 | +\newcommand{\SPIDissuersWithMentionsFirstYear}{1{,}224} | ||||
| 26 | +\newcommand{\SPIDissuersWithMentionsLastFullYear}{2{,}280} | ||||
| 27 | +\newcommand{\SPIDmentionsPerIssuerFirstYear}{9.7} | ||||
| 28 | +\newcommand{\SPIDmentionsPerIssuerLastFullYear}{10.2} | ||||
| 29 | +\newcommand{\SPIDlastYear}{2026} | ||||
| 30 | +\newcommand{\SPIDcallsOneZeroK}{135{,}310} | ||||
| 31 | +\newcommand{\SPIDcallsOneZeroQ}{363{,}973} | ||||
| 32 | +\newcommand{\SPIDcallsEightK}{204{,}442} | ||||
| 33 | +\newcommand{\SPIDledgerTotal}{1{,}123{,}098} | ||||
| 34 | +\newcommand{\SPIDledgerMentionsReturned}{308{,}477} | ||||
| 35 | +\newcommand{\SPIDledgerGap}{412} | ||||
| 36 | +\newcommand{\SPIDledgerGapByForm}{10-K 127, 10-Q 257, 8-K 28} | ||||
| 37 | +\newcommand{\SPIDmentionsStoredEightK}{74{,}566} | ||||
| 38 | +\newcommand{\SPIDmentionsLedgerEightK}{74{,}594} | ||||
| 39 | +\newcommand{\SPIDledgerSectionsEightK}{400{,}601} | ||||
| 40 | +\newcommand{\SPIDmentionsStoredOneZeroK}{81{,}495} | ||||
| 41 | +\newcommand{\SPIDmentionsLedgerOneZeroK}{81{,}622} | ||||
| 42 | +\newcommand{\SPIDledgerSectionsOneZeroK}{143{,}704} | ||||
| 43 | +\newcommand{\SPIDmentionsStoredOneZeroQ}{152{,}004} | ||||
| 44 | +\newcommand{\SPIDmentionsLedgerOneZeroQ}{152{,}261} | ||||
| 45 | +\newcommand{\SPIDledgerSectionsOneZeroQ}{578{,}793} | ||||
| 46 | +\newcommand{\SPIDproductiveSections}{168{,}230} | ||||
| 47 | +\newcommand{\SPIDproductiveShareLedger}{15.0} | ||||
| 48 | +\newcommand{\SPIDmentionsPerLedgerSection}{0.27} | ||||
| 49 | +\newcommand{\SPIDmentionsPerProductiveSection}{1.83} | ||||
| 50 | +\newcommand{\SPIDgatePassTotal}{703{,}725} | ||||
| 51 | +\newcommand{\SPIDproductiveShareCalls}{23.9} | ||||
| 52 | +\newcommand{\SPIDmentionsPerCall}{0.44} | ||||
| 53 | +\newcommand{\SPIDtenKmdnaMentions}{39{,}099} | ||||
| 54 | +\newcommand{\SPIDtenQmdnaMentions}{99{,}305} | ||||
| 55 | +\newcommand{\SPIDtenKbusinessMentions}{30{,}492} | ||||
| 56 | +\newcommand{\SPIDtenQfinancialStatementsMentions}{30{,}159} | ||||
| 57 | +\newcommand{\SPIDtenKfullBodyRows}{1{,}230} | ||||
| 58 | +\newcommand{\SPIDtypesPctSum}{100.1} | ||||
| 59 | +\newcommand{\SPIDamountMedianMusd}{128.5} | ||||
| 60 | +\newcommand{\SPIDsmallNThreshold}{50} | ||||
| 61 | +\newcommand{\SPIDsmallNCategories}{ai_initiative, automation, cloud_migration} | ||||
| 62 | +\newcommand{\SPIDshareDataCenter}{0.4} | ||||
| 63 | +\newcommand{\SPIDnDataCenter}{449} | ||||
| 64 | +\newcommand{\SPIDwithAmountDataCenter}{129} | ||||
| 65 | +\newcommand{\SPIDmedianAmountDataCenter}{166.0} | ||||
| 66 | +\newcommand{\SPIDshareAiInitiative}{0.3} | ||||
| 67 | +\newcommand{\SPIDnAiInitiative}{376} | ||||
| 68 | +\newcommand{\SPIDwithAmountAiInitiative}{20} | ||||
| 69 | +\newcommand{\SPIDmedianAmountAiInitiative}{127.5} | ||||
| 70 | +\newcommand{\SPIDshareCloudMigration}{0.2} | ||||
| 71 | +\newcommand{\SPIDnCloudMigration}{264} | ||||
| 72 | +\newcommand{\SPIDwithAmountCloudMigration}{35} | ||||
| 73 | +\newcommand{\SPIDmedianAmountCloudMigration}{50.0} | ||||
| 74 | +\newcommand{\SPIDshareAutomation}{0.3} | ||||
| 75 | +\newcommand{\SPIDnAutomation}{374} | ||||
| 76 | +\newcommand{\SPIDwithAmountAutomation}{25} | ||||
| 77 | +\newcommand{\SPIDmedianAmountAutomation}{13.7} | ||||
| 78 | +\newcommand{\SPIDshareMaIntegration}{34.5} | ||||
| 79 | +\newcommand{\SPIDnMaIntegration}{41{,}742} | ||||
| 80 | +\newcommand{\SPIDwithAmountMaIntegration}{28{,}044} | ||||
| 81 | +\newcommand{\SPIDmedianAmountMaIntegration}{155.0} | ||||
| 82 | +\newcommand{\SPIDshareCapexProgram}{8.9} | ||||
| 83 | +\newcommand{\SPIDnCapexProgram}{10{,}720} | ||||
| 84 | +\newcommand{\SPIDwithAmountCapexProgram}{9{,}452} | ||||
| 85 | +\newcommand{\SPIDmedianAmountCapexProgram}{300.0} | ||||
| 86 | +\newcommand{\SPIDsharePartnership}{8.2} | ||||
| 87 | +\newcommand{\SPIDnPartnership}{9{,}909} | ||||
| 88 | +\newcommand{\SPIDwithAmountPartnership}{2{,}943} | ||||
| 89 | +\newcommand{\SPIDmedianAmountPartnership}{75.0} | ||||
| 90 | +\newcommand{\SPIDshareGeographicExpansion}{6.5} | ||||
| 91 | +\newcommand{\SPIDnGeographicExpansion}{7{,}868} | ||||
| 92 | +\newcommand{\SPIDwithAmountGeographicExpansion}{1{,}431} | ||||
| 93 | +\newcommand{\SPIDmedianAmountGeographicExpansion}{36.7} | ||||
| 94 | +\newcommand{\SPIDsharePlantConstruction}{6.3} | ||||
| 95 | +\newcommand{\SPIDnPlantConstruction}{7{,}590} | ||||
| 96 | +\newcommand{\SPIDwithAmountPlantConstruction}{2{,}511} | ||||
| 97 | +\newcommand{\SPIDmedianAmountPlantConstruction}{56.0} | ||||
| 98 | +\newcommand{\SPIDtechCategoriesCountRange}{264--449} | ||||
| 99 | +\newcommand{\SPIDsectorTopFourPerIssuerRange}{62--75} | ||||
| 100 | +\newcommand{\SPIDsectorTopFourNames}{utilities, materials, energy, real estate} | ||||
| 101 | +\newcommand{\SPIDsectorFinancialsPerIssuer}{32} | ||||
| 102 | +\newcommand{\SPIDsectorProjectsHealthCare}{23{,}679} | ||||
| 103 | +\newcommand{\SPIDsectorIssuersHealthCare}{429} | ||||
| 104 | +\newcommand{\SPIDsectorProjectsIndustrials}{17{,}890} | ||||
| 105 | +\newcommand{\SPIDsectorIssuersIndustrials}{386} | ||||
| 106 | +\newcommand{\SPIDspanKnownN}{103{,}500} | ||||
| 107 | +\newcommand{\SPIDspanUnknownN}{17{,}599} | ||||
| 108 | +\newcommand{\SPIDspanUnknownShareAll}{14.5} | ||||
| 109 | +\newcommand{\SPIDspanZeroShareAll}{55.5} | ||||
| 110 | +\newcommand{\SPIDspanZeroShareKnown}{64.9} | ||||
| 111 | +\newcommand{\SPIDspanOverFiveShareAll}{8.5} | ||||
| 112 | +\newcommand{\SPIDspanOverFiveShareKnown}{10.0} | ||||
| 113 | +\newcommand{\SPIDstatusShareInProgress}{41.6} | ||||
| 114 | +\newcommand{\SPIDstatusShareCompleted}{41.0} | ||||
| 115 | +\newcommand{\SPIDstatusSharePlanned}{15.1} | ||||
| 116 | +\newcommand{\SPIDstatusShareMentioned}{2.3} | ||||
| 117 | +\newcommand{\SPIDsingleMentionShare}{60.2} | ||||
| 118 | +\newcommand{\SPIDoverTenMentionShare}{3.1} | ||||
| 119 | +\newcommand{\SPIDconfShareOne}{35.2} | ||||
| 120 | +\newcommand{\SPIDconfShareNine}{32.7} | ||||
| 121 | +\newcommand{\SPIDconfShareLeSix}{1.6} | ||||
| 122 | +\newcommand{\SPIDamountMentions}{120{,}240} | ||||
| 123 | +\newcommand{\SPIDamountMentionsShare}{39.0} | ||||
| 124 | +\newcommand{\SPIDamountProjects}{56{,}500} | ||||
| 125 | +\newcommand{\SPIDamountProjectsShare}{46.7} | ||||
| 126 | +\newcommand{\SPIDamountPTwoFiveMusd}{23} | ||||
| 127 | +\newcommand{\SPIDamountPSevenFiveMusd}{500} | ||||
| 128 | +\newcommand{\SPIDamountPNineNineBusd}{12.5} | ||||
| 129 | +\newcommand{\SPIDamountSumTusd}{55.9} | ||||
| 130 | +\newcommand{\SPIDkgNodes}{183{,}472} | ||||
| 131 | +\newcommand{\SPIDkgEdges}{204{,}456} | ||||
| 132 | +\newcommand{\SPIDkgNodesProject}{121{,}099} | ||||
| 133 | +\newcommand{\SPIDkgNodesPartner}{36{,}822} | ||||
| 134 | +\newcommand{\SPIDkgNodesLocation}{14{,}508} | ||||
| 135 | +\newcommand{\SPIDkgNodesTechnology}{8{,}551} | ||||
| 136 | +\newcommand{\SPIDkgNodesCompany}{2{,}492} | ||||
| 137 | +\newcommand{\SPIDprovenanceForms}{8-K} | ||||
| 138 | +\newcommand{\SPIDprovenanceN}{8} | ||||
| 139 | +\newcommand{\SPIDprovenanceFirst}{2014-04-08} | ||||
| 140 | +\newcommand{\SPIDprovenanceLast}{2022-03-07} | ||||
| 141 | +\newcommand{\SPIDprovenanceMaxAmountBusd}{11.5} | ||||
| 142 | +\newcommand{\SPIDprovenanceNameYear}{2022} | ||||
| 143 | +\newcommand{\SPIDgateRegexFlags}{re.IGNORECASE} | ||||
| 144 | +\newcommand{\SPIDgateRegexLength}{215} | ||||
| 145 | +\newcommand{\SPIDgateRegexAlternatives}{25} | ||||
| 146 | +\newcommand{\SPIDtaxonomyN}{21} | ||||
| 147 | +\newcommand{\SPIDcallsTotal}{703{,}725} | ||||
| 148 | +\newcommand{\SPIDgatePassSections}{703{,}737} | ||||
| 149 | +\newcommand{\SPIDgateRejectedSections}{419{,}361} | ||||
| 150 | +\newcommand{\SPIDledgerUnderThirtyWords}{1{,}070} | ||||
| 151 | +\newcommand{\SPIDusdPerCall}{0.0013} | ||||
| 152 | +\newcommand{\SPIDoctoberCalls}{608{,}439} | ||||
| 153 | +\newcommand{\SPIDoctoberRows}{968{,}134} | ||||
| 154 | +\newcommand{\SPIDoctoberWallClockHours}{13.7} | ||||
| 155 | +\newcommand{\SPIDoutageHours}{5.3} | ||||
| 156 | +\newcommand{\SPIDoctoberEffectiveHours}{8.4} | ||||
| 157 | +\newcommand{\SPIDoctoberRowsPerSecondEffective}{31.9} | ||||
| 158 | +\newcommand{\SPIDoctoberCallsPerSecondEffective}{20.1} | ||||
| 159 | +\newcommand{\SPIDoctoberFirstFlush}{2026-10-27 21:35} | ||||
| 160 | +\newcommand{\SPIDoctoberLastFlush}{2026-10-28 11:15} | ||||
| 161 | +\newcommand{\SPIDbackfilledRows}{154{,}964} | ||||
| 162 | +\newcommand{\SPIDjuneLogSections}{153{,}894} | ||||
| 163 | +\newcommand{\SPIDbackfilledUnderThirtyWords}{1{,}070} | ||||
| 164 | +\newcommand{\SPIDbackfilledMentions}{39{,}930} | ||||
| 165 | +\newcommand{\SPIDbackfilledCiks}{500} | ||||
| 166 | +\newcommand{\SPIDbackfilledGateFail}{59{,}666} | ||||
| 167 | +\newcommand{\SPIDoutageWindowStart}{2026-10-28 01:10} | ||||
| 168 | +\newcommand{\SPIDoutageWindowEnd}{2026-10-28 06:22} | ||||
| 169 | +\newcommand{\SPIDllmCallFailedTotal}{94{,}609} | ||||
| 170 | +\newcommand{\SPIDllmCallFailedInsideOutage}{94{,}467} | ||||
| 171 | +\newcommand{\SPIDllmCallFailedOutsideOutage}{142} | ||||
| 172 | +\newcommand{\SPIDllmFailedJsonDecodeError}{14} | ||||
| 173 | +\newcommand{\SPIDllmFailedRetryableHttpFourTwoNineFivexx}{94{,}593} | ||||
| 174 | +\newcommand{\SPIDllmFailedConnectionError}{2} | ||||
| 175 | +\newcommand{\SPIDjuneLlmCallFailed}{3} | ||||
| 176 | +\newcommand{\SPIDcollisionSections}{304} | ||||
| 177 | +\newcommand{\SPIDcollisionMentions}{412} | ||||
| 178 | +\newcommand{\SPIDprojectsKeyedGeneral}{229} | ||||
| 179 | +\newcommand{\SPIDmentionsKeyedGeneral}{2{,}152} | ||||
| 180 | +\newcommand{\SPIDprojectsKeyedNameToken}{66{,}405} | ||||
| 181 | +\newcommand{\SPIDprojectsKeyedNameTokenPct}{54.8} | ||||
| 182 | +\newcommand{\SPIDprojectsKeyedLocation}{54{,}658} | ||||
| 183 | +\newcommand{\SPIDprojectsKeyedLocationPct}{45.1} | ||||
| 184 | +\newcommand{\SPIDprojectsKeyedGeneralPct}{0.2} | ||||
| 185 | +\newcommand{\SPIDprojectsTieMaxConfidence}{33{,}820} | ||||
| 186 | +\newcommand{\SPIDprojectsTieDistinctNames}{20{,}118} | ||||
| 187 | +\newcommand{\SPIDprojectsStatusTie}{972} | ||||
| 188 | +\newcommand{\SPIDmentionsAtConfidenceOne}{108{,}023} | ||||
| 189 | +\newcommand{\SPIDprojectCountRangeKeys}{21{,}027--308{,}065} | ||||
| 190 | +\newcommand{\SPIDprojectCountRangeKeysN}{9} | ||||
| 191 | +\newcommand{\SPIDprojectCountRangeAll}{16{,}562--308{,}065} | ||||
| 192 | +\newcommand{\SPIDprojectCountCurrentKey}{121{,}099} | ||||
| 193 | +\newcommand{\SPIDtopNameTokens}{\texttt{acquisition} (3.4\%), \texttt{share} (3.1\%), \texttt{expansion} (1.8\%), \texttt{stock} (1.7\%), \texttt{investment} (1.6\%), \texttt{merger} (1.3\%), \texttt{sale} (1.3\%), \texttt{development} (1.2\%), \texttt{strategic} (0.9\%), \texttt{capital} (0.8\%)} | ||||
| 194 | +\newcommand{\SPIDtopNameTokensProjectsPct}{17.2} | ||||
| 195 | +\newcommand{\SPIDprojects}{121{,}099} | ||||
| 196 | +\newcommand{\SPIDmentions}{308{,}065} | ||||
| 197 | +\newcommand{\SPIDissuersWithProjects}{2{,}492} | ||||
| 198 | +\newcommand{\SPIDissuersWithFilings}{2{,}499} | ||||
| 199 | +\newcommand{\SPIDsectionsTotal}{1{,}342{,}283} | ||||
| 200 | +\newcommand{\SPIDchunks}{10{,}720{,}890} | ||||
| 201 | +\newcommand{\SPIDprojectsWithLocation}{54{,}583} | ||||
| 202 | +\newcommand{\SPIDprojectsTokenKeyedShare}{54.9} | ||||
| 203 | +\newcommand{\SPIDprojectsPerIssuerMean}{48.6} | ||||
| 204 | +\newcommand{\SPIDprojectsPerIssuerMedian}{42} | ||||
| 205 | +\newcommand{\SPIDextractedAtFirst}{2026-10-27 13:12:32} | ||||
| 206 | +\newcommand{\SPIDextractedAtLast}{2026-10-28 11:15:49} | ||||
| 207 | +\newcommand{\SPIDbrief}{the research-design brief, i.e. the specification of required tables, keys and validation gates against which \texttt{docs/implementation\_audit.md} audits the build} | ||||
| 208 | +\newcommand{\SPIDnewProjectsPerIssuerFirstYear}{6.4} | ||||
| 209 | +\newcommand{\SPIDnewProjectsPerIssuerLastFullYear}{3.5} | ||||
modified
paper/tables/prompt_verbatim.tex
+21 −1
@@ -20,7 +20,27 @@ Return STRICT JSON: {"projects": [ ... ]}. Each project object has: | |||||
| 20 | confidence : 0.0-1.0 (how clearly this is a real, concrete project) | 20 | confidence : 0.0-1.0 (how clearly this is a real, concrete project) | ||
| 21 | 21 | ||||
| 22 | Allowed project_type values: | 22 | Allowed project_type values: | ||
| 23 | − - <code>: <label> (the 21 taxonomy entries of Table A1) | 23 | + - manufacturing_expansion: Manufacturing / Capacity Expansion | ||
| 24 | + - plant_construction: Plant / Site Construction | ||||
| 25 | + - data_center: Data Center | ||||
| 26 | + - cloud_migration: Cloud Migration | ||||
| 27 | + - ai_initiative: AI / ML Initiative | ||||
| 28 | + - digital_transformation: Digital Transformation | ||||
| 29 | + - erp_implementation: ERP / Systems Implementation | ||||
| 30 | + - rd_program: R&D Program | ||||
| 31 | + - product_launch: New Product / Launch | ||||
| 32 | + - drug_pipeline: Drug / Clinical Pipeline | ||||
| 33 | + - ma_integration: M&A / Integration | ||||
| 34 | + - geographic_expansion: Geographic Expansion | ||||
| 35 | + - capex_program: Capital Expenditure Program | ||||
| 36 | + - supply_chain: Supply Chain Initiative | ||||
| 37 | + - automation: Automation / Robotics | ||||
| 38 | + - sustainability: Sustainability / ESG | ||||
| 39 | + - energy_transition: Energy Transition / Renewables | ||||
| 40 | + - infrastructure: Infrastructure Investment | ||||
| 41 | + - cost_reduction: Cost Reduction / Restructuring | ||||
| 42 | + - partnership: Strategic Partnership | ||||
| 43 | + - technology_deployment: Technology Deployment | ||||
| 24 | 44 | ||||
| 25 | Rules: | 45 | Rules: | ||
| 26 | - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | 46 | - Extract ONLY real initiatives, not generic risk-factor boilerplate or hypotheticals. | ||
modified
paper/tables/tab_amounts.tex
+1 −1
@@ -5,4 +5,4 @@ | |||||
| 5 | Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | 5 | Largest amount cited per project{,} M USD ($N$ = 56{{,}}500) & 3.2 & 23 & 128.5 & 500 & 1{,}503 & 12{,}501 \\ | ||
| 6 | Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | 6 | Projects per issuer ($N$ = 2{{,}}492) & 12 & 24 & 42 & 67 & 94 & 140 \\ | ||
| 7 | \bottomrule | 7 | \bottomrule | ||
| 8 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 8 | +\end{tabular} | ||||
added
paper/tables/tab_amounts_note.tex
+1 −0
@@ -0,0 +1 @@ | |||||
| 1 | +\item Amounts are the largest USD figure cited across the mentions of a project; they mix transaction values, budgets, contract values and estimates and are never summed in the text. | ||||
modified
paper/tables/tab_confidence.tex
+1 −1
@@ -9,4 +9,4 @@ Self-reported confidence (rounded) & Mentions \\ | |||||
| 9 | 0.9 & 100{,}831 \\ | 9 | 0.9 & 100{,}831 \\ | ||
| 10 | 1.0 & 108{,}418 \\ | 10 | 1.0 & 108{,}418 \\ | ||
| 11 | \bottomrule | 11 | \bottomrule | ||
| 12 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 12 | +\end{tabular} | ||||
modified
paper/tables/tab_coverage_year.tex
+20 −20
@@ -1,23 +1,23 @@ | |||||
| 1 | −\begin{tabular}{lrrrrrrr} | 1 | +\begin{tabular}{lrrrrrrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Projects first seen \\ | 3 | +Year & Issuers & 10-K & 10-Q & 8-K & Mentions & Issuers w/ mentions & Mentions per issuer & Projects first seen \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 7{,}835 \\ | 5 | +2010 & 1{,}392 & 1{,}357 & 4{,}152 & 17{,}672 & 11{,}908 & 1{,}224 & 9.7 & 7{,}835 \\ | ||
| 6 | −2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 6{,}679 \\ | 6 | +2011 & 1{,}424 & 1{,}409 & 4{,}260 & 18{,}533 & 13{,}551 & 1{,}287 & 10.5 & 6{,}679 \\ | ||
| 7 | −2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 6{,}444 \\ | 7 | +2012 & 1{,}472 & 1{,}436 & 4{,}363 & 19{,}437 & 14{,}302 & 1{,}333 & 10.7 & 6{,}444 \\ | ||
| 8 | −2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 6{,}286 \\ | 8 | +2013 & 1{,}539 & 1{,}483 & 4{,}530 & 19{,}773 & 14{,}611 & 1{,}408 & 10.4 & 6{,}286 \\ | ||
| 9 | −2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 6{,}732 \\ | 9 | +2014 & 1{,}619 & 1{,}549 & 4{,}780 & 20{,}927 & 16{,}274 & 1{,}490 & 10.9 & 6{,}732 \\ | ||
| 10 | −2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 6{,}675 \\ | 10 | +2015 & 1{,}689 & 1{,}635 & 4{,}983 & 22{,}077 & 17{,}073 & 1{,}535 & 11.1 & 6{,}675 \\ | ||
| 11 | −2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 6{,}240 \\ | 11 | +2016 & 1{,}744 & 1{,}692 & 5{,}146 & 22{,}447 & 16{,}429 & 1{,}579 & 10.4 & 6{,}240 \\ | ||
| 12 | −2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 6{,}490 \\ | 12 | +2017 & 1{,}810 & 1{,}745 & 5{,}359 & 22{,}897 & 17{,}350 & 1{,}636 & 10.6 & 6{,}490 \\ | ||
| 13 | −2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 6{,}815 \\ | 13 | +2018 & 1{,}893 & 1{,}827 & 5{,}582 & 23{,}709 & 18{,}064 & 1{,}737 & 10.4 & 6{,}815 \\ | ||
| 14 | −2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 6{,}837 \\ | 14 | +2019 & 1{,}973 & 1{,}903 & 5{,}829 & 24{,}256 & 18{,}737 & 1{,}806 & 10.4 & 6{,}837 \\ | ||
| 15 | −2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 7{,}212 \\ | 15 | +2020 & 2{,}089 & 1{,}985 & 6{,}095 & 27{,}424 & 17{,}970 & 1{,}919 & 9.4 & 7{,}212 \\ | ||
| 16 | −2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 8{,}216 \\ | 16 | +2021 & 2{,}247 & 2{,}116 & 6{,}606 & 27{,}944 & 21{,}312 & 2{,}077 & 10.3 & 8{,}216 \\ | ||
| 17 | −2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 8{,}158 \\ | 17 | +2022 & 2{,}291 & 2{,}258 & 6{,}830 & 26{,}533 & 22{,}720 & 2{,}105 & 10.8 & 8{,}158 \\ | ||
| 18 | −2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 7{,}977 \\ | 18 | +2023 & 2{,}329 & 2{,}295 & 6{,}938 & 26{,}851 & 22{,}955 & 2{,}129 & 10.8 & 7{,}977 \\ | ||
| 19 | −2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 7{,}819 \\ | 19 | +2024 & 2{,}387 & 2{,}341 & 7{,}092 & 27{,}438 & 22{,}503 & 2{,}204 & 10.2 & 7{,}819 \\ | ||
| 20 | −2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 7{,}990 \\ | 20 | +2025 & 2{,}453 & 2{,}406 & 7{,}300 & 28{,}716 & 23{,}220 & 2{,}280 & 10.2 & 7{,}990 \\ | ||
| 21 | −2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 6{,}694 \\ | 21 | +2026 & 2{,}498 & 2{,}338 & 5{,}322 & 23{,}972 & 19{,}086 & 2{,}260 & 8.4 & 6{,}694 \\ | ||
| 22 | \bottomrule | 22 | \bottomrule | ||
| 23 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 23 | +\end{tabular} | ||||
added
paper/tables/tab_datamodel.tex
+24 −0
@@ -0,0 +1,24 @@ | |||||
| 1 | +\begin{longtable}{p{3.1cm}p{8.6cm}p{4.3cm}} | ||||
| 2 | +\caption{Tables required by the research design, current implementation and status}\label{tab:datamodel}\\ | ||||
| 3 | +\toprule | ||||
| 4 | +Required table & Current implementation (database, table, key fields) & Status \\ | ||||
| 5 | +\midrule | ||||
| 6 | +\endfirsthead | ||||
| 7 | +\toprule | ||||
| 8 | +Required table & Current implementation (database, table, key fields) & Status \\ | ||||
| 9 | +\midrule | ||||
| 10 | +\endhead | ||||
| 11 | +\bottomrule | ||||
| 12 | +\endfoot | ||||
| 13 | +\texttt{universe\_snapshots} & \texttt{data/metadata/universe.json} + \texttt{companies} (CIK, ticker(s), name, sector, flags \texttt{in\_sp500}/\texttt{in\_russell3000}, fund weight, SIC, state, exchange, FYE, shard); snapshot date and source URL recorded & Partial: no file hash, no membership dates, no match confidence \\ | ||||
| 14 | +\texttt{issuers} & \texttt{companies} (one row per CIK; former names from SEC header stored in metadata JSON only) & Partial: no name history, no parent/subsidiary links \\ | ||||
| 15 | +\texttt{filings} & \texttt{filings} (accession PK, CIK FK, form, filing/report dates, SEC URL, local path, size, download and parse status) & Implemented; no acceptance timestamp, no checksum \\ | ||||
| 16 | +\texttt{sections} & \texttt{filing\_sections} (UUID PK, accession FK, name, item, order, text, counts) & Implemented; no character offsets into raw HTML \\ | ||||
| 17 | +\texttt{project\_mentions} & \texttt{project\_mentions} (SHA-1 id, accession, CIK, section id, name, type, description, objective, amount, raw amount, lists, years, status, confidence, backend, issuer context) & Implemented; no evidence span, no prompt hash, no five-way label \\ | ||||
| 18 | +\texttt{projects} & \texttt{projects} (MD5 of CIK$|$type$|$location-or-name, representative fields, max amount, counts, first/last seen and cited years) & Implemented (deterministic key) \\ | ||||
| 19 | +\texttt{project\_mention\_links} & implicit in the key (every mention maps to exactly one project) & Proposed: explicit table with score, resolver version, adjudication \\ | ||||
| 20 | +\texttt{project\_events} & \texttt{project\_timeline} (one row per mention: date, form, status, amount, confidence, snippet) & Partial: mention rows, not typed events \\ | ||||
| 21 | +\texttt{project\_amounts} & \texttt{amount\_usd}/\texttt{amount\_raw} on mentions; max on projects & Proposed: type, currency, scale, basis, realization flag \\ | ||||
| 22 | +\texttt{extraction\_runs} & \texttt{extracted\_sections} ledger (section, accession, backend, n\_mentions, timestamp) + run logs & Partial: no prompt/schema/taxonomy hashes, no call status \\ | ||||
| 23 | +\texttt{human\_labels} & --- & Proposed \\ | ||||
| 24 | +\end{longtable} | ||||
modified
paper/tables/tab_filings.tex
+1 −1
@@ -8,4 +8,4 @@ Form & Filings & First filing & Last filing & Raw HTML (GB) & Not parsed \\ | |||||
| 8 | \midrule | 8 | \midrule | ||
| 9 | Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | 9 | Total & 527{,}548 & 2010-01-04 & 2026-10-27 & 254.5 & 16 \\ | ||
| 10 | \bottomrule | 10 | \bottomrule | ||
| 11 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 11 | +\end{tabular} | ||||
added
paper/tables/tab_fp_classes.tex
+67 −0
@@ -0,0 +1,67 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +form\_type & item & project\_type & mentions & matched & pct & matched\_amount\_gusd & total\_amount\_gusd \\ | ||||
| 4 | +\midrule | ||||
| 5 | +8-K & full\_body & ma\_integration & 41{,}635 & 6{,}055 & 14.54 & 4{,}431.64 & 21{,}674.49 \\ | ||||
| 6 | +10-Q & md\_and\_a & ma\_integration & 32{,}011 & 319 & 1.0 & 297.7 & 15{,}025.4 \\ | ||||
| 7 | +10-Q & financial\_statements & ma\_integration & 19{,}165 & 260 & 1.36 & 158.12 & 7{,}994.15 \\ | ||||
| 8 | +10-K & md\_and\_a & ma\_integration & 14{,}027 & 60 & 0.43 & 159.74 & 6{,}818.94 \\ | ||||
| 9 | +10-Q & md\_and\_a & drug\_pipeline & 12{,}341 & 1 & 0.01 & & 161.52 \\ | ||||
| 10 | +10-K & business & ma\_integration & 10{,}554 & 21 & 0.2 & 3.23 & 7{,}258.55 \\ | ||||
| 11 | +8-K & full\_body & capex\_program & 10{,}415 & 7{,}135 & 68.51 & 5{,}436.73 & 7{,}873.23 \\ | ||||
| 12 | +10-Q & md\_and\_a & geographic\_expansion & 7{,}602 & 2 & 0.03 & & 461.42 \\ | ||||
| 13 | +10-Q & md\_and\_a & plant\_construction & 6{,}624 & 1 & 0.02 & 3.0 & 882.35 \\ | ||||
| 14 | +10-Q & md\_and\_a & product\_launch & 5{,}935 & 1 & 0.02 & & 119.1 \\ | ||||
| 15 | +10-Q & md\_and\_a & partnership & 5{,}563 & 38 & 0.68 & 16.44 & 2{,}199.1 \\ | ||||
| 16 | +10-Q & md\_and\_a & rd\_program & 5{,}221 & 0 & 0.0 & & 169.15 \\ | ||||
| 17 | +8-K & full\_body & partnership & 4{,}919 & 170 & 3.46 & 127.29 & 1{,}238.44 \\ | ||||
| 18 | +10-Q & unregistered\_sales & ma\_integration & 4{,}565 & 2{,}334 & 51.13 & 1{,}995.02 & 2{,}281.79 \\ | ||||
| 19 | +10-Q & md\_and\_a & cost\_reduction & 4{,}437 & 111 & 2.5 & 79.52 & 571.57 \\ | ||||
| 20 | +8-K & full\_body & drug\_pipeline & 4{,}308 & 4 & 0.09 & 0.25 & 118.97 \\ | ||||
| 21 | +10-Q & md\_and\_a & capex\_program & 4{,}088 & 856 & 20.94 & 796.27 & 2{,}998.47 \\ | ||||
| 22 | +10-K & md\_and\_a & drug\_pipeline & 3{,}992 & 0 & 0.0 & & 28.52 \\ | ||||
| 23 | +10-Q & md\_and\_a & manufacturing\_expansion & 3{,}870 & 3 & 0.08 & & 182.76 \\ | ||||
| 24 | +10-Q & controls\_and\_procedures & ma\_integration & 3{,}536 & 5 & 0.14 & 0.43 & 248.21 \\ | ||||
| 25 | +8-K & full\_body & cost\_reduction & 3{,}361 & 643 & 19.13 & 266.15 & 807.44 \\ | ||||
| 26 | +10-K & properties & plant\_construction & 3{,}266 & 0 & 0.0 & & 132.04 \\ | ||||
| 27 | +10-K & md\_and\_a & geographic\_expansion & 3{,}156 & 2 & 0.06 & & 363.7 \\ | ||||
| 28 | +10-K & business & drug\_pipeline & 3{,}141 & 0 & 0.0 & & 25.43 \\ | ||||
| 29 | +10-Q & md\_and\_a & infrastructure & 3{,}047 & 4 & 0.13 & 1.85 & 3{,}464.59 \\ | ||||
| 30 | +10-K & business & geographic\_expansion & 2{,}833 & 0 & 0.0 & & 658.32 \\ | ||||
| 31 | +10-Q & controls\_and\_procedures & erp\_implementation & 2{,}703 & 0 & 0.0 & & 61.61 \\ | ||||
| 32 | +10-Q & md\_and\_a & digital\_transformation & 2{,}484 & 0 & 0.0 & & 67.68 \\ | ||||
| 33 | +10-K & business & product\_launch & 2{,}462 & 0 & 0.0 & & 51.71 \\ | ||||
| 34 | +10-Q & md\_and\_a & technology\_deployment & 2{,}410 & 35 & 1.45 & 16.79 & 179.42 \\ | ||||
| 35 | +10-K & md\_and\_a & plant\_construction & 2{,}310 & 3 & 0.13 & 0.34 & 286.23 \\ | ||||
| 36 | +10-K & md\_and\_a & product\_launch & 2{,}285 & 0 & 0.0 & & 80.96 \\ | ||||
| 37 | +10-K & md\_and\_a & partnership & 2{,}246 & 16 & 0.71 & 7.36 & 223.79 \\ | ||||
| 38 | +10-Q & risk\_factors & ma\_integration & 2{,}133 & 20 & 0.94 & 10.78 & 962.24 \\ | ||||
| 39 | +10-K & md\_and\_a & rd\_program & 2{,}001 & 0 & 0.0 & & 125.19 \\ | ||||
| 40 | +8-K & full\_body & plant\_construction & 1{,}983 & 23 & 1.16 & 4.29 & 685.84 \\ | ||||
| 41 | +10-K & business & rd\_program & 1{,}898 & 0 & 0.0 & & 186.29 \\ | ||||
| 42 | +10-K & business & partnership & 1{,}880 & 1 & 0.05 & 0.3 & 132.91 \\ | ||||
| 43 | +10-Q & unregistered\_sales & capex\_program & 1{,}740 & 1{,}419 & 81.55 & 1{,}841.69 & 1{,}972.31 \\ | ||||
| 44 | +10-Q & financial\_statements & drug\_pipeline & 1{,}735 & 0 & 0.0 & & 9.85 \\ | ||||
| 45 | +10-K & business & plant\_construction & 1{,}671 & 1 & 0.06 & 0.51 & 104.63 \\ | ||||
| 46 | +10-Q & financial\_statements & partnership & 1{,}664 & 56 & 3.37 & 35.31 & 185.07 \\ | ||||
| 47 | +8-K & full\_body & infrastructure & 1{,}651 & 24 & 1.45 & 15.03 & 540.82 \\ | ||||
| 48 | +10-K & md\_and\_a & cost\_reduction & 1{,}564 & 23 & 1.47 & 15.4 & 216.18 \\ | ||||
| 49 | +10-K & md\_and\_a & capex\_program & 1{,}507 & 372 & 24.68 & 392.53 & 1{,}848.99 \\ | ||||
| 50 | +10-Q & financial\_statements & cost\_reduction & 1{,}448 & 159 & 10.98 & 112.79 & 201.78 \\ | ||||
| 51 | +10-K & md\_and\_a & manufacturing\_expansion & 1{,}392 & 3 & 0.22 & 0.02 & 107.1 \\ | ||||
| 52 | +10-Q & financial\_statements & plant\_construction & 1{,}385 & 6 & 0.43 & 2.18 & 168.42 \\ | ||||
| 53 | +10-Q & unregistered\_sales & cost\_reduction & 1{,}381 & 1{,}212 & 87.76 & 1{,}020.25 & 1{,}033.64 \\ | ||||
| 54 | +8-K & full\_body & geographic\_expansion & 1{,}286 & 12 & 0.93 & 1.71 & 122.93 \\ | ||||
| 55 | +10-K & business & technology\_deployment & 1{,}265 & 8 & 0.63 & 10.52 & 86.41 \\ | ||||
| 56 | +10-K & properties & geographic\_expansion & 1{,}219 & 2 & 0.16 & 0.49 & 59.64 \\ | ||||
| 57 | +8-K & full\_body & technology\_deployment & 1{,}177 & 202 & 17.16 & 126.57 & 317.1 \\ | ||||
| 58 | +10-K & md\_and\_a & digital\_transformation & 1{,}090 & 0 & 0.0 & & 12.71 \\ | ||||
| 59 | +10-K & md\_and\_a & infrastructure & 1{,}067 & 3 & 0.28 & 2.16 & 1{,}525.2 \\ | ||||
| 60 | +10-K & business & digital\_transformation & 1{,}050 & 0 & 0.0 & & 84.4 \\ | ||||
| 61 | +10-K & business & manufacturing\_expansion & 1{,}025 & 0 & 0.0 & & 62.95 \\ | ||||
| 62 | +10-K & md\_and\_a & technology\_deployment & 978 & 27 & 2.76 & 13.61 & 52.45 \\ | ||||
| 63 | +8-K & full\_body & rd\_program & 968 & 4 & 0.41 & 0.75 & 217.38 \\ | ||||
| 64 | +10-K & properties & manufacturing\_expansion & 913 & 0 & 0.0 & & 11.22 \\ | ||||
| 65 | +\multicolumn{8}{l}{\ldots 232 more rows in \texttt{results/fp\_classes/financing\_regex\_by\_item\_type.csv}} \\ | ||||
| 66 | +\bottomrule | ||||
| 67 | +\end{tabular} | ||||
modified
paper/tables/tab_graph.tex
+1 −1
@@ -11,4 +11,4 @@ Nodes & \texttt{Location} & 14{,}508 \\ | |||||
| 11 | Nodes & \texttt{Technology} & 8{,}551 \\ | 11 | Nodes & \texttt{Technology} & 8{,}551 \\ | ||
| 12 | Nodes & \texttt{Company} & 2{,}492 \\ | 12 | Nodes & \texttt{Company} & 2{,}492 \\ | ||
| 13 | \bottomrule | 13 | \bottomrule | ||
| 14 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 14 | +\end{tabular} | ||||
added
paper/tables/tab_grounding.tex
+67 −0
@@ -0,0 +1,67 @@ | |||||
| 1 | +\begin{tabular}{lrrrrrrrr} | ||||
| 2 | +\toprule | ||||
| 3 | +form & item & field & n\_mentions & n\_sections & n\_grounded & grounded\_share & ci\_low & ci\_high \\ | ||||
| 4 | +\midrule | ||||
| 5 | +10-Q & financial\_statements & years & 27{,}098 & 18{,}213 & 27{,}085 & 1 & 0.999 & 1 \\ | ||||
| 6 | +10-K & md\_and\_a & years & 30{,}493 & 13{,}345 & 30{,}433 & 0.998 & 0.997 & 0.999 \\ | ||||
| 7 | +10-Q & md\_and\_a & years & 76{,}041 & 36{,}503 & 75{,}894 & 0.998 & 0.998 & 0.998 \\ | ||||
| 8 | +8-K & full\_body & years & 69{,}075 & 59{,}295 & 68{,}920 & 0.998 & 0.997 & 0.998 \\ | ||||
| 9 | +10-Q & risk\_factors & years & 2{,}905 & 2{,}288 & 2{,}891 & 0.995 & 0.992 & 0.997 \\ | ||||
| 10 | +10-Q & unregistered\_sales & years & 9{,}009 & 6{,}923 & 8{,}999 & 0.999 & 0.998 & 1 \\ | ||||
| 11 | +10-K & properties & years & 6{,}061 & 3{,}401 & 6{,}009 & 0.991 & 0.988 & 0.994 \\ | ||||
| 12 | +10-K & business & years & 20{,}877 & 9{,}200 & 20{,}776 & 0.995 & 0.994 & 0.996 \\ | ||||
| 13 | +10-K & quantitative\_disclosures & years & 460 & 330 & 460 & 1.0 & 1.0 & 1.0 \\ | ||||
| 14 | +10-Q & controls\_and\_procedures & years & 6{,}606 & 5{,}434 & 6{,}571 & 0.995 & 0.993 & 0.996 \\ | ||||
| 15 | +10-Q & legal\_proceedings & years & 679 & 558 & 677 & 0.997 & 0.993 & 1.0 \\ | ||||
| 16 | +10-Q & quantitative\_disclosures & years & 697 & 531 & 693 & 0.994 & 0.984 & 1.0 \\ | ||||
| 17 | +10-K & risk\_factors & years & 1{,}704 & 1{,}034 & 1{,}695 & 0.995 & 0.991 & 0.998 \\ | ||||
| 18 | +10-Q & full\_document & years & 93 & 34 & 93 & 1.0 & 1.0 & 1.0 \\ | ||||
| 19 | +10-K & full\_body & years & 671 & 301 & 669 & 0.997 & 0.993 & 1.0 \\ | ||||
| 20 | +8-K & full\_body & locations & 39{,}399 & 36{,}198 & 26{,}714 & 0.678 & 0.673 & 0.683 \\ | ||||
| 21 | +10-Q & md\_and\_a & locations & 32{,}007 & 19{,}345 & 27{,}070 & 0.846 & 0.841 & 0.851 \\ | ||||
| 22 | +10-K & md\_and\_a & locations & 12{,}521 & 7{,}249 & 10{,}572 & 0.844 & 0.836 & 0.852 \\ | ||||
| 23 | +10-Q & financial\_statements & locations & 11{,}964 & 9{,}096 & 9{,}225 & 0.771 & 0.762 & 0.78 \\ | ||||
| 24 | +10-Q & controls\_and\_procedures & locations & 691 & 618 & 485 & 0.702 & 0.665 & 0.735 \\ | ||||
| 25 | +10-K & properties & locations & 6{,}769 & 3{,}524 & 5{,}818 & 0.86 & 0.847 & 0.872 \\ | ||||
| 26 | +10-K & risk\_factors & locations & 863 & 578 & 744 & 0.862 & 0.832 & 0.892 \\ | ||||
| 27 | +10-K & business & locations & 11{,}424 & 6{,}107 & 9{,}918 & 0.868 & 0.86 & 0.876 \\ | ||||
| 28 | +10-Q & risk\_factors & locations & 1{,}063 & 828 & 917 & 0.863 & 0.838 & 0.886 \\ | ||||
| 29 | +10-Q & quantitative\_disclosures & locations & 166 & 136 & 133 & 0.801 & 0.732 & 0.861 \\ | ||||
| 30 | +10-Q & unregistered\_sales & locations & 723 & 595 & 533 & 0.737 & 0.701 & 0.774 \\ | ||||
| 31 | +10-K & quantitative\_disclosures & locations & 165 & 124 & 138 & 0.836 & 0.764 & 0.899 \\ | ||||
| 32 | +10-Q & legal\_proceedings & locations & 595 & 463 & 375 & 0.63 & 0.584 & 0.674 \\ | ||||
| 33 | +10-Q & full\_document & locations & 86 & 31 & 76 & 0.884 & 0.811 & 0.951 \\ | ||||
| 34 | +10-K & full\_body & locations & 347 & 184 & 314 & 0.905 & 0.864 & 0.939 \\ | ||||
| 35 | +10-K & md\_and\_a & partners & 6{,}268 & 4{,}289 & 6{,}223 & 0.993 & 0.99 & 0.995 \\ | ||||
| 36 | +10-Q & financial\_statements & partners & 5{,}933 & 4{,}753 & 5{,}902 & 0.995 & 0.993 & 0.997 \\ | ||||
| 37 | +8-K & full\_body & partners & 42{,}167 & 37{,}852 & 41{,}998 & 0.996 & 0.995 & 0.997 \\ | ||||
| 38 | +10-Q & md\_and\_a & partners & 16{,}157 & 11{,}346 & 15{,}993 & 0.99 & 0.988 & 0.992 \\ | ||||
| 39 | +10-Q & risk\_factors & partners & 1{,}359 & 1{,}189 & 1{,}352 & 0.995 & 0.991 & 0.998 \\ | ||||
| 40 | +10-Q & controls\_and\_procedures & partners & 488 & 453 & 487 & 0.998 & 0.994 & 1.0 \\ | ||||
| 41 | +10-Q & unregistered\_sales & partners & 1{,}964 & 1{,}599 & 1{,}939 & 0.987 & 0.981 & 0.993 \\ | ||||
| 42 | +10-K & business & partners & 5{,}164 & 3{,}484 & 5{,}132 & 0.994 & 0.991 & 0.996 \\ | ||||
| 43 | +10-K & properties & partners & 799 & 531 & 792 & 0.991 & 0.985 & 0.997 \\ | ||||
| 44 | +10-Q & legal\_proceedings & partners & 313 & 282 & 308 & 0.984 & 0.968 & 0.997 \\ | ||||
| 45 | +10-K & risk\_factors & partners & 403 & 326 & 399 & 0.99 & 0.979 & 0.998 \\ | ||||
| 46 | +10-Q & quantitative\_disclosures & partners & 98 & 91 & 97 & 0.99 & 0.965 & 1.0 \\ | ||||
| 47 | +10-K & quantitative\_disclosures & partners & 64 & 54 & 64 & 1.0 & 1.0 & 1.0 \\ | ||||
| 48 | +10-K & full\_body & partners & 127 & 89 & 127 & 1.0 & 1.0 & 1.0 \\ | ||||
| 49 | +10-Q & full\_document & partners & 16 & 10 & 16 & 1.0 & 1.0 & 1.0 \\ | ||||
| 50 | +10-K & md\_and\_a & technologies & 5{,}311 & 3{,}624 & 3{,}252 & 0.612 & 0.596 & 0.627 \\ | ||||
| 51 | +10-Q & controls\_and\_procedures & technologies & 2{,}014 & 1{,}957 & 1{,}820 & 0.904 & 0.89 & 0.917 \\ | ||||
| 52 | +10-Q & financial\_statements & technologies & 3{,}218 & 2{,}625 & 1{,}968 & 0.612 & 0.591 & 0.632 \\ | ||||
| 53 | +10-Q & unregistered\_sales & technologies & 232 & 188 & 116 & 0.5 & 0.429 & 0.574 \\ | ||||
| 54 | +10-K & business & technologies & 5{,}448 & 3{,}379 & 3{,}654 & 0.671 & 0.655 & 0.686 \\ | ||||
| 55 | +8-K & full\_body & technologies & 5{,}116 & 4{,}613 & 2{,}977 & 0.582 & 0.567 & 0.596 \\ | ||||
| 56 | +10-Q & md\_and\_a & technologies & 13{,}596 & 9{,}327 & 8{,}492 & 0.625 & 0.615 & 0.635 \\ | ||||
| 57 | +10-Q & risk\_factors & technologies & 549 & 483 & 328 & 0.597 & 0.552 & 0.645 \\ | ||||
| 58 | +10-K & properties & technologies & 616 & 416 & 367 & 0.596 & 0.545 & 0.647 \\ | ||||
| 59 | +10-Q & quantitative\_disclosures & technologies & 51 & 46 & 23 & 0.451 & 0.302 & 0.591 \\ | ||||
| 60 | +10-K & risk\_factors & technologies & 367 & 297 & 223 & 0.608 & 0.549 & 0.665 \\ | ||||
| 61 | +10-Q & legal\_proceedings & technologies & 63 & 60 & 47 & 0.746 & 0.63 & 0.854 \\ | ||||
| 62 | +10-K & quantitative\_disclosures & technologies & 47 & 39 & 32 & 0.681 & 0.535 & 0.814 \\ | ||||
| 63 | +10-Q & full\_document & technologies & 6 & 6 & 1 & 0.167 & 0.0 & 0.5 \\ | ||||
| 64 | +10-K & full\_body & technologies & 106 & 71 & 61 & 0.576 & 0.462 & 0.68 \\ | ||||
| 65 | +\multicolumn{9}{l}{\ldots 30 more rows in \texttt{results/grounding/grounding\_by\_field\_form\_item.csv}} \\ | ||||
| 66 | +\bottomrule | ||||
| 67 | +\end{tabular} | ||||
modified
paper/tables/tab_items.tex
+17 −17
@@ -1,21 +1,21 @@ | |||||
| 1 | −\begin{tabular}{llrr} | 1 | +\begin{tabular}{llrrr} | ||
| 2 | \toprule | 2 | \toprule | ||
| 3 | −Form & Section (item) & Sections extracted & Mentions \\ | 3 | +Form & Section (item) & Ledgered sections & Mentions & Gate passed (calls) \\ | ||
| 4 | \midrule | 4 | \midrule | ||
| 5 | −10-K & \texttt{business} & 29{,}285 & 30{,}492 \\ | 5 | +10-K & \texttt{business} & 29{,}285 & 30{,}492 & 29{,}118 \\ | ||
| 6 | −10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 \\ | 6 | +10-K & \texttt{risk\_factors} & 29{,}158 & 2{,}797 & 28{,}941 \\ | ||
| 7 | −10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 \\ | 7 | +10-K & \texttt{md\_and\_a} & 28{,}919 & 39{,}099 & 28{,}050 \\ | ||
| 8 | −10-K & \texttt{properties} & 28{,}236 & 7{,}879 \\ | 8 | +10-K & \texttt{properties} & 28{,}236 & 7{,}879 & 25{,}940 \\ | ||
| 9 | −10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 \\ | 9 | +10-K & \texttt{quantitative\_disclosures} & 26{,}876 & 531 & 22{,}032 \\ | ||
| 10 | −10-K & \texttt{full\_body} & 1{,}230 & 824 \\ | 10 | +10-K & \texttt{full\_body} & 1{,}230 & 824 & 1{,}229 \\ | ||
| 11 | \midrule | 11 | \midrule | ||
| 12 | −10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 \\ | 12 | +10-Q & \texttt{controls\_and\_procedures} & 92{,}393 & 7{,}082 & 20{,}913 \\ | ||
| 13 | −10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 \\ | 13 | +10-Q & \texttt{financial\_statements} & 91{,}882 & 30{,}159 & 90{,}106 \\ | ||
| 14 | −10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 \\ | 14 | +10-Q & \texttt{md\_and\_a} & 91{,}371 & 99{,}305 & 90{,}269 \\ | ||
| 15 | −10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 \\ | 15 | +10-Q & \texttt{quantitative\_disclosures} & 83{,}138 & 808 & 46{,}720 \\ | ||
| 16 | −10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 \\ | 16 | +10-Q & \texttt{unregistered\_sales} & 80{,}210 & 9{,}535 & 61{,}867 \\ | ||
| 17 | −10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 \\ | 17 | +10-Q & \texttt{risk\_factors} & 72{,}560 & 4{,}420 & 33{,}826 \\ | ||
| 18 | −10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 \\ | 18 | +10-Q & \texttt{legal\_proceedings} & 66{,}998 & 845 & 20{,}031 \\ | ||
| 19 | −10-Q & \texttt{full\_document} & 241 & 107 \\ | 19 | +10-Q & \texttt{full\_document} & 241 & 107 & 241 \\ | ||
| 20 | \bottomrule | 20 | \bottomrule | ||
| 21 | −\end{tabular} | ||||
| \ No newline at end of file | |||||
| 21 | +\end{tabular} | ||||
added
paper/tables/tab_june_october.tex
+15 −0
Large diff (15 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_overmerge.tex
+27 −0
Large diff (27 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_provenance.tex
+1 −1
Large diff (6 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_reconciliation.tex
+2 −2
Large diff (14 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_sections.tex
+7 −7
Large diff (19 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_sections_note.tex
+1 −0
Large diff (1 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_sectors.tex
+1 −1
Large diff (6 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_spec_curve.tex
+51 −0
Large diff (51 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_status.tex
+1 −1
Large diff (6 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_table3_calls.tex
+9 −0
Large diff (9 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_timeseries_cohort.tex
+40 −0
Large diff (40 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_top_issuers.tex
+1 −1
Large diff (6 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_top_name_tokens.tex
+27 −0
Large diff (27 lines) is not loaded by default to keep the page fast.Load diff
modified
paper/tables/tab_types.tex
+25 −25
Large diff (55 lines) is not loaded by default to keep the page fast.Load diff
added
paper/tables/tab_types_note.tex
+1 −0
Large diff (1 lines) is not loaded by default to keep the page fast.Load diff
added
results/inline_numbers.json
+1037 −0
Large diff (1,038 lines) is not loaded by default to keep the page fast.Load diff
added
results/tables_diff.txt
+318 −0
Large diff (319 lines) is not loaded by default to keep the page fast.Load diff