N4: two runs, two 10-K segmenters, cohort regression, fiscal_year audit, 10-K funnel, git/mtime evidence
Items: FIRC0001-AI_C-07 EDIT-04 EDIT-19 EDIT-20 ADVE-02 ADVE-11 AI_C-03 THEO-06 DATA-13 ECON-03 ECON-08. No June git revision exists (first commit 0304133 on 2026-10-27); extractor files mtime 2026-06-17; df7905d..2324b58 = store.py +3/-1. Legacy MD&A carries 141 vs 272 '$' tokens (92% of pairs fewer); 86 full_document 10-Ks (35 invisible, 51 via legacy); 10-K funnel 341,207 -> 116,540 + 27,164 = 143,704 exact; cohort dummy +8.8 (all runs) vs -23.0 (October only). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +20,001 −0
added
code/audit/a04_git_evidence.sh
+75 −0
@@ -0,0 +1,75 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# N4 (a) - read-only git / file-system evidence on M3U96b:~/UQO_RESEARCH_DB for the two extraction runs. | |
| 3 | +# Writes results/runs/git_evidence.md and the two inter-revision patches. Never modifies git state. | |
| 4 | +set -u | |
| 5 | +OUT="$HOME/firc0001_review/results/runs"; mkdir -p "$OUT" | |
| 6 | +cd "$HOME/UQO_RESEARCH_DB" || exit 1 | |
| 7 | +export GIT_PAGER=cat | |
| 8 | +FILES="project-db/spid/extract/llm.py project-db/spid/extract/rules.py project-db/spid/extract/base.py project-db/spid/taxonomy.py project-db/spid/resolve.py project-db/spid/ingest.py project-db/spid/config.py project-db/spid/pipeline.py project-db/spid/store.py project-db/spid/__main__.py project-db/run_full_extract.py src/parsing/html_parser.py src/parsing/section_extractor.py src/parsing/filing_parser.py src/parsing/chunker.py scripts/parse_filings.py" | |
| 9 | +{ | |
| 10 | +echo "# Git and file-system evidence for the two SPID extraction runs (N4a)" | |
| 11 | +echo | |
| 12 | +echo "Generated $(date -Iseconds) on $(hostname) from a read-only inspection of \`~/UQO_RESEARCH_DB\` (clone of \`gitsrv:srv/git/uqo-research-db.git\`)." | |
| 13 | +echo | |
| 14 | +echo "## Repository state" | |
| 15 | +echo '```' | |
| 16 | +echo "HEAD: $(git rev-parse HEAD)"; echo "branch: $(git rev-parse --abbrev-ref HEAD)"; echo "tags: $(git tag | tr '\n' ' ')" | |
| 17 | +echo "remote: $(git remote -v | head -1)" | |
| 18 | +echo "working tree (git status --short | head -20):"; git status --short | head -20 | |
| 19 | +echo '```' | |
| 20 | +echo | |
| 21 | +echo "## All commits (there is no commit before 2026-10-27; the June 2026 run has no git revision)" | |
| 22 | +echo '```' | |
| 23 | +git log --format='%h %ad %an %s' --date=iso | |
| 24 | +echo '```' | |
| 25 | +echo | |
| 26 | +echo "## First commit of each extractor / parser file (git log --diff-filter=A)" | |
| 27 | +echo '```' | |
| 28 | +for f in $FILES; do printf '%-45s %s\n' "$f" "$(git log --diff-filter=A --format='%h %ad' --date=iso -- "$f" | tail -1)"; done | |
| 29 | +echo '```' | |
| 30 | +echo | |
| 31 | +echo "## Commits touching each file" | |
| 32 | +echo '```' | |
| 33 | +for f in $FILES; do printf '%-45s %s\n' "$f" "$(git log --format='%h' -- "$f" | tr '\n' ' ')"; done | |
| 34 | +echo '```' | |
| 35 | +echo | |
| 36 | +echo "## File modification times (file-system evidence; extractor modules untouched since 2026-06-17)" | |
| 37 | +echo '```' | |
| 38 | +for f in $FILES; do [ -f "$f" ] && printf '%-45s %s %s bytes sha256=%s\n' "$f" "$(stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$f")" "$(stat -f '%z' "$f")" "$(shasum -a 256 "$f" | cut -c1-16)"; done | |
| 39 | +echo '```' | |
| 40 | +echo | |
| 41 | +echo "## git diff --stat 0304133..2324b58 (first commit -> revision cited by the paper) for the extractor, parser and driver" | |
| 42 | +echo '```' | |
| 43 | +git diff --stat 0304133 2324b58 -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py | |
| 44 | +echo '```' | |
| 45 | +echo | |
| 46 | +echo "## git diff --stat df7905d..2324b58 (the pair named in the paper)" | |
| 47 | +echo '```' | |
| 48 | +git diff --stat df7905d 2324b58 -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py | |
| 49 | +echo '```' | |
| 50 | +echo | |
| 51 | +echo "## git diff --stat 2324b58..HEAD (after the paper's revision)" | |
| 52 | +echo '```' | |
| 53 | +git diff --stat 2324b58 HEAD -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py | |
| 54 | +echo '```' | |
| 55 | +echo | |
| 56 | +echo "## June 2026 run parameters from project-db/logs/extract.log (first and last lines; model id absent)" | |
| 57 | +echo '```' | |
| 58 | +head -2 project-db/logs/extract.log; echo '...'; tail -2 project-db/logs/extract.log | |
| 59 | +echo "grep -c 'gpt' project-db/logs/extract.log = $(grep -c 'gpt' project-db/logs/extract.log)" | |
| 60 | +echo "grep -c 'gpt' data/logs/spid_extract.log = $(grep -c 'gpt' data/logs/spid_extract.log)" | |
| 61 | +echo "project-db/logs mtimes:"; for f in project-db/logs/*; do printf ' %-40s %s\n' "$f" "$(stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$f")"; done | |
| 62 | +echo "project-db/.env present: $( [ -f project-db/.env ] && echo yes || echo no ) (content never read)" | |
| 63 | +echo "project-db/.env.example model line: $(grep -h 'SPID_LLM_MODEL' project-db/.env.example 2>/dev/null | head -2 | tr '\n' ' ')" | |
| 64 | +echo '```' | |
| 65 | +echo | |
| 66 | +echo "## Interpretation" | |
| 67 | +echo "- The repository's first commit (0304133, 2026-10-27) already contains \`project-db/\`; no revision of the June 2026 code exists in git. The sentence '2324b58 supersedes df7905d examined in the June study' names two October commits." | |
| 68 | +echo "- The extractor modules (llm.py, rules.py, base.py, taxonomy.py, resolve.py, ingest.py) carry 2026-06-17 modification times: the prompt, the gate, the taxonomy and the resolver used in October are, by file-system evidence, the files that existed before the June run. The driver/config/store (config.py, pipeline.py, __main__.py: 2026-10-27 12:34; store.py: 2026-10-27 21:34) were edited before the first commit, so their June versions are not recoverable." | |
| 69 | +echo "- The only change to these files between df7905d and 2324b58 is the one-line DuckDB 2.0 compatibility fix in store.py (see patch)." | |
| 70 | +} > "$OUT/git_evidence.md" | |
| 71 | +git diff 0304133 2324b58 -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py > "$OUT/git_diff_0304133_2324b58.patch" | |
| 72 | +git diff df7905d 2324b58 -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py > "$OUT/git_diff_df7905d_2324b58.patch" | |
| 73 | +git diff 2324b58 HEAD -- project-db/spid src/parsing scripts/parse_filings.py project-db/run_full_extract.py > "$OUT/git_diff_2324b58_HEAD.patch" | |
| 74 | +wc -l "$OUT"/*.patch | |
| 75 | +echo "git state after: $(git rev-parse HEAD) $(git status --short | wc -l) changed paths" | |
added
code/audit/a04_runs_segmenters.py
+202 −0
@@ -0,0 +1,202 @@ | ||
| 1 | +"""N4 - two runs, two 10-K segmenters, cohort confounding, fiscal_year audit, 10-K funnel (cluster D). Read-only.""" | |
| 2 | +from __future__ import annotations | |
| 3 | +import json, sys | |
| 4 | +from pathlib import Path | |
| 5 | +import numpy as np | |
| 6 | +sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| 7 | +from common import * | |
| 8 | + | |
| 9 | +r = Run("runs") | |
| 10 | +con = connect(SPID_DB) | |
| 11 | +tmin = con.execute("select min(extracted_at) from extracted_sections").fetchone()[0] | |
| 12 | +con.execute(f"""CREATE TEMP TABLE led AS | |
| 13 | +SELECT e.section_id, e.accession_number, e.n_mentions, CASE WHEN e.extracted_at = TIMESTAMP '{tmin}' THEN 'backfilled_june' ELSE 'october' END AS run, | |
| 14 | + f.form_type, f.cik, f.filing_date, f.fiscal_year AS filing_fiscal_year, co.in_sp500, co.in_russell3000 | |
| 15 | +FROM extracted_sections e JOIN src_filings f ON f.accession_number = e.accession_number LEFT JOIN src_companies co ON f.cik = co.cik""") | |
| 16 | +con.execute("""CREATE TEMP TABLE secmeta AS | |
| 17 | +SELECT section_id, section_name, word_count, 'parsed' AS segmenter FROM src_sections | |
| 18 | +UNION ALL SELECT section_id, section_name, word_count, 'legacy_ingest' FROM spid_sections""") | |
| 19 | +con.execute("CREATE TEMP TABLE ledm AS SELECT l.*, s.section_name, s.word_count, s.segmenter FROM led l LEFT JOIN secmeta s USING (section_id)") | |
| 20 | + | |
| 21 | +# ---- (b) legacy (ingest.py) vs parsed (src/parsing) text of the same 10-K items ---------------- | |
| 22 | +MONEY = r"\$\s?\d[\d,]*(\.\d+)?\s?(billion|million|thousand|bn|mm)?" | |
| 23 | +YEAR = r"\b(19[89]\d|20[0-3]\d)\b" | |
| 24 | +feat = lambda src: f"""SELECT section_id, accession_number, section_name, word_count, length(text) AS nchar, | |
| 25 | + length(text) - length(replace(text, '$', '')) AS dollar_tokens, | |
| 26 | + len(regexp_extract_all(text, '{YEAR}')) AS year_tokens, | |
| 27 | + len(regexp_extract_all(text, '{MONEY}')) AS money_tokens, | |
| 28 | + len(regexp_extract_all(text, '\\b\\d[\\d,]*(\\.\\d+)?\\b')) AS numeric_tokens | |
| 29 | +FROM {src}""" | |
| 30 | +print("features: legacy", flush=True) | |
| 31 | +con.execute(f"CREATE TEMP TABLE leg AS {feat('spid_sections')}") | |
| 32 | +print("features: parsed 10-K of the same accessions", flush=True) | |
| 33 | +con.execute(f"CREATE TEMP TABLE par AS {feat('src_sections')} WHERE accession_number IN (SELECT DISTINCT accession_number FROM spid_sections)") | |
| 34 | +r.dump("segmenter_item_comparison.csv", con, """ | |
| 35 | +WITH j AS (SELECT l.section_name, l.word_count lw, p.word_count pw, l.dollar_tokens ld, p.dollar_tokens pd, l.year_tokens ly, p.year_tokens py, l.money_tokens lm, p.money_tokens pm, l.numeric_tokens ln, p.numeric_tokens pn | |
| 36 | + FROM leg l JOIN par p ON l.accession_number = p.accession_number AND l.section_name = p.section_name) | |
| 37 | +SELECT section_name, count(*) pairs, | |
| 38 | + round(median(lw)) legacy_median_words, round(median(pw)) parsed_median_words, | |
| 39 | + round(quantile_cont(lw*1.0/nullif(pw,0), 0.10),3) ratio_p10, round(quantile_cont(lw*1.0/nullif(pw,0), 0.50),3) ratio_p50, round(quantile_cont(lw*1.0/nullif(pw,0), 0.90),3) ratio_p90, | |
| 40 | + round(avg(ld),1) legacy_mean_dollar, round(avg(pd),1) parsed_mean_dollar, | |
| 41 | + round(avg(ly),1) legacy_mean_years, round(avg(py),1) parsed_mean_years, | |
| 42 | + round(avg(lm),1) legacy_mean_money, round(avg(pm),1) parsed_mean_money, | |
| 43 | + round(avg(ln),1) legacy_mean_numeric, round(avg(pn),1) parsed_mean_numeric, | |
| 44 | + round(100.0*count(*) FILTER (WHERE lm < pm)/count(*),2) pct_legacy_fewer_money_tokens, | |
| 45 | + round(100.0*count(*) FILTER (WHERE lm = 0 AND pm > 0)/count(*),2) pct_legacy_zero_money_parsed_positive, | |
| 46 | + round(100.0*count(*) FILTER (WHERE ld < pd)/count(*),2) pct_legacy_fewer_dollar, | |
| 47 | + round(100.0*count(*) FILTER (WHERE lw < 0.5*pw)/count(*),2) pct_legacy_lt_half_words, | |
| 48 | + round(100.0*count(*) FILTER (WHERE lw > 2*pw)/count(*),2) pct_legacy_gt_double_words | |
| 49 | +FROM j GROUP BY 1 ORDER BY 2 DESC""") | |
| 50 | +S = {} | |
| 51 | +S["legacy_accessions"] = con.execute("select count(distinct accession_number) from spid_sections").fetchone()[0] | |
| 52 | +S["legacy_rows"] = con.execute("select count(*) from spid_sections").fetchone()[0] | |
| 53 | +S["legacy_accessions_with_parsed_counterpart"] = con.execute("select count(distinct accession_number) from par").fetchone()[0] | |
| 54 | +S["legacy_rows_matched_by_item"] = con.execute("select count(*) from leg l join par p on l.accession_number=p.accession_number and l.section_name=p.section_name").fetchone()[0] | |
| 55 | +S["legacy_section_names"] = con.execute("select section_name, count(*), round(median(word_count)) from spid_sections group by 1 order by 2 desc").fetchall() | |
| 56 | +S["parsed_section_names_same_accessions"] = con.execute("select section_name, count(*), round(median(word_count)) from par group by 1 order by 2 desc").fetchall() | |
| 57 | +S["legacy_items_per_accession"] = con.execute("select n, count(*) from (select accession_number, count(*) n from spid_sections group by 1) group by 1 order by 1").fetchall() | |
| 58 | +S["legacy_accessions_missing_item"] = con.execute("""select section_name, count(*) from (select a.accession_number, i.section_name from (select distinct accession_number from spid_sections) a cross join (select unnest(['business','risk_factors','properties','md_and_a','quantitative_disclosures']) section_name) i | |
| 59 | + where not exists (select 1 from spid_sections s where s.accession_number=a.accession_number and s.section_name=i.section_name)) group by 1 order by 2 desc""").fetchall() | |
| 60 | + | |
| 61 | +# ---- (c) full_body rows ---------------------------------------------------------------------------- | |
| 62 | +S["full_body_rows"] = con.execute("select count(*), count(distinct accession_number), round(median(word_count)), min(word_count), max(word_count) from spid_sections where section_name='full_body'").fetchone() | |
| 63 | +S["full_body_accessions_with_other_items"] = con.execute("select count(distinct accession_number) from spid_sections where section_name='full_body' and accession_number in (select accession_number from spid_sections where section_name<>'full_body')").fetchone()[0] | |
| 64 | +S["full_body_ledger_yield"] = con.execute("""select count(*), count(*) filter (where n_mentions>0), sum(n_mentions), round(sum(n_mentions)*1.0/count(*),4), round(100.0*count(*) filter (where n_mentions>0)/count(*),2) | |
| 65 | + from ledm where segmenter='legacy_ingest' and section_name='full_body'""").fetchone() | |
| 66 | +S["legacy_item_ledger_yield"] = con.execute("""select section_name, count(*), count(*) filter (where n_mentions>0), sum(n_mentions), round(sum(n_mentions)*1.0/count(*),4), round(100.0*count(*) filter (where n_mentions>0)/count(*),2), round(median(word_count)) | |
| 67 | + from ledm where segmenter='legacy_ingest' group by 1 order by 2 desc""").fetchall() | |
| 68 | +S["parsed_10k_item_ledger_yield"] = con.execute("""select section_name, count(*), count(*) filter (where n_mentions>0), sum(n_mentions), round(sum(n_mentions)*1.0/count(*),4), round(100.0*count(*) filter (where n_mentions>0)/count(*),2), round(median(word_count)) | |
| 69 | + from ledm where segmenter='parsed' and form_type='10-K' group by 1 order by 2 desc""").fetchall() | |
| 70 | +S["full_body_accessions_parsed_items_available"] = con.execute("""select count(distinct p.accession_number), count(*) from par p where p.accession_number in (select accession_number from spid_sections where section_name='full_body') and p.section_name in ('business','risk_factors','properties','md_and_a','quantitative_disclosures')""").fetchone() | |
| 71 | + | |
| 72 | +# ---- (d) parser-fallback full_document 10-Ks never submitted --------------------------------------------- | |
| 73 | +S["full_document_10k"] = con.execute("""select count(*), count(distinct s.accession_number) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name='full_document'""").fetchone() | |
| 74 | +S["full_document_10k_in_legacy"] = con.execute("""select count(distinct s.accession_number) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name='full_document' and s.accession_number in (select accession_number from spid_sections)""").fetchone()[0] | |
| 75 | +S["full_document_10k_in_ledger"] = con.execute("""select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name='full_document' and s.section_id in (select section_id from extracted_sections)""").fetchone()[0] | |
| 76 | +S["full_document_10k_accessions_with_no_other_section"] = con.execute("""select count(*) from (select s.accession_number from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' group by 1 having bool_and(s.section_name='full_document'))""").fetchone()[0] | |
| 77 | +S["full_document_10k_invisible_accessions"] = con.execute("""select count(*) from (select s.accession_number from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.accession_number not in (select accession_number from spid_sections) group by 1 having bool_and(s.section_name='full_document'))""").fetchone()[0] | |
| 78 | +S["full_document_by_form"] = con.execute("""select f.form_type, count(*), count(distinct s.accession_number), count(*) filter (where s.section_id in (select section_id from extracted_sections)) in_ledger from src_sections s join src_filings f using(accession_number) where s.section_name='full_document' group by 1""").fetchall() | |
| 79 | +S["sections_10k_by_item_all_parsed"] = con.execute("""select s.section_name, count(*), round(median(s.word_count)), count(*) filter (where s.word_count<30) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' group by 1 order by 2 desc""").fetchall() | |
| 80 | +r.write_json("segmenters.json", S) | |
| 81 | + | |
| 82 | +# ---- (e) within S&P 500, within form: backfilled vs October ---------------------------------------------- | |
| 83 | +r.dump("within_sp500_by_run_form.csv", con, """ | |
| 84 | +SELECT run, form_type, count(*) rows, count(DISTINCT accession_number) filings, count(DISTINCT cik) issuers, | |
| 85 | + count(*) FILTER (WHERE n_mentions>0) with_mentions, sum(n_mentions) mentions, | |
| 86 | + round(100.0*count(*) FILTER (WHERE n_mentions>0)/count(*),2) productive_pct, round(sum(n_mentions)*1.0/count(*),4) mentions_per_row, | |
| 87 | + round(median(word_count)) median_words, min(filing_date) first_filing, max(filing_date) last_filing | |
| 88 | +FROM ledm WHERE in_sp500 GROUP BY 1,2 ORDER BY 2,1""") | |
| 89 | +r.dump("within_sp500_10q_by_item_run.csv", con, """ | |
| 90 | +SELECT section_name item, run, count(*) rows, count(*) FILTER (WHERE n_mentions>0) with_mentions, sum(n_mentions) mentions, | |
| 91 | + round(100.0*count(*) FILTER (WHERE n_mentions>0)/count(*),2) productive_pct, round(sum(n_mentions)*1.0/count(*),4) mentions_per_row, round(median(word_count)) median_words | |
| 92 | +FROM ledm WHERE in_sp500 AND form_type='10-Q' GROUP BY 1,2 ORDER BY 1,2""") | |
| 93 | +con.execute("""CREATE TEMP TABLE pm AS SELECT m.*, l.run, l.in_sp500, l.in_russell3000 FROM project_mentions m JOIN led l USING (section_id)""") | |
| 94 | +r.dump("within_sp500_10q_mentions_by_run.csv", con, """ | |
| 95 | +SELECT run, count(*) mentions, round(avg(confidence),4) mean_conf, round(median(confidence),3) median_conf, | |
| 96 | + round(100.0*count(*) FILTER (WHERE confidence>=0.9)/count(*),2) pct_conf_ge_09, round(100.0*count(*) FILTER (WHERE amount_usd IS NOT NULL)/count(*),2) pct_with_amount, | |
| 97 | + round(100.0*count(*) FILTER (WHERE len(locations)>0)/count(*),2) pct_with_location, round(100.0*count(*) FILTER (WHERE len(partners)>0)/count(*),2) pct_with_partner, | |
| 98 | + round(100.0*count(*) FILTER (WHERE len(years)>0)/count(*),2) pct_with_years, round(avg(length(description)),1) mean_desc_chars | |
| 99 | +FROM pm WHERE in_sp500 AND form_type='10-Q' GROUP BY 1""") | |
| 100 | +r.dump("within_sp500_10q_type_mix_by_run.csv", con, """ | |
| 101 | +SELECT project_type, count(*) FILTER (WHERE run='backfilled_june') june_n, count(*) FILTER (WHERE run='october') oct_n, | |
| 102 | + round(100.0*count(*) FILTER (WHERE run='backfilled_june')/sum(count(*) FILTER (WHERE run='backfilled_june')) OVER (),2) june_pct, | |
| 103 | + round(100.0*count(*) FILTER (WHERE run='october')/sum(count(*) FILTER (WHERE run='october')) OVER (),2) oct_pct | |
| 104 | +FROM pm WHERE in_sp500 AND form_type='10-Q' GROUP BY 1 ORDER BY 2+3 DESC""") | |
| 105 | +r.dump("mentions_by_run_form_all.csv", con, """ | |
| 106 | +SELECT run, form_type, count(*) mentions, round(avg(confidence),4) mean_conf, round(100.0*count(*) FILTER (WHERE amount_usd IS NOT NULL)/count(*),2) pct_with_amount, | |
| 107 | + round(100.0*count(*) FILTER (WHERE len(locations)>0)/count(*),2) pct_with_location, round(100.0*count(*) FILTER (WHERE fiscal_year IS NULL)/count(*),2) pct_fiscal_year_null | |
| 108 | +FROM pm GROUP BY 1,2 ORDER BY 2,1""") | |
| 109 | +# paired within-issuer comparison (10-Q, S&P 500 issuers present in both runs) | |
| 110 | +pairs = con.execute(""" | |
| 111 | +WITH a AS (SELECT cik, run, count(*) rows, count(*) FILTER (WHERE n_mentions>0) prod, sum(n_mentions) men FROM ledm WHERE in_sp500 AND form_type='10-Q' GROUP BY 1,2) | |
| 112 | +SELECT j.cik, j.rows, j.prod, j.men, o.rows, o.prod, o.men FROM a j JOIN a o ON j.cik=o.cik AND j.run='backfilled_june' AND o.run='october'""").fetchall() | |
| 113 | +P = {"issuers_paired": len(pairs)} | |
| 114 | +if pairs: | |
| 115 | + d_prod = np.array([o_p/o_r - j_p/j_r for _, j_r, j_p, j_m, o_r, o_p, o_m in pairs]) | |
| 116 | + d_men = np.array([o_m/o_r - j_m/j_r for _, j_r, j_p, j_m, o_r, o_p, o_m in pairs]) | |
| 117 | + P["mean_diff_productive_share_oct_minus_june"] = float(d_prod.mean()); P["se"] = float(d_prod.std(ddof=1)/np.sqrt(len(d_prod))) | |
| 118 | + P["mean_diff_mentions_per_row_oct_minus_june"] = float(d_men.mean()); P["se_m"] = float(d_men.std(ddof=1)/np.sqrt(len(d_men))) | |
| 119 | + P["june_mean_productive_share"] = float(np.mean([j_p/j_r for _, j_r, j_p, *_ in pairs])); P["oct_mean_productive_share"] = float(np.mean([o_p/o_r for _, j_r, j_p, j_m, o_r, o_p, o_m in pairs])) | |
| 120 | +r.write_json("within_issuer_paired_10q.json", P) | |
| 121 | + | |
| 122 | +# ---- (f) Table 6 within cohort + cohort dummy regression ------------------------------------------------ | |
| 123 | +con.execute("""CREATE TEMP TABLE _m2 AS SELECT *, lower(coalesce(try(locations[1]), '')) AS loc1, | |
| 124 | + regexp_extract(lower(regexp_replace(project_name, '[^A-Za-z ]', '', 'g')), '([a-z]{4,})', 1) AS name_tok FROM pm""") | |
| 125 | +con.execute("""CREATE TEMP TABLE k2 AS SELECT *, cik || '|' || project_type || '|' || CASE WHEN loc1 <> '' THEN loc1 ELSE coalesce(nullif(name_tok,''),'general') END AS gkey FROM _m2""") | |
| 126 | +grp = "CASE WHEN in_sp500 AND in_russell3000 THEN 'both' WHEN in_sp500 THEN 'sp500_only' WHEN in_russell3000 THEN 'russell_only' ELSE 'none' END" | |
| 127 | +r.dump("table6_cohort.csv", con, f""" | |
| 128 | +WITH allm AS (SELECT {grp} g, count(DISTINCT gkey) projects, count(DISTINCT cik) issuers, count(*) mentions FROM k2 GROUP BY 1), | |
| 129 | + octm AS (SELECT {grp} g, count(DISTINCT gkey) projects, count(DISTINCT cik) issuers, count(*) mentions FROM k2 WHERE run='october' GROUP BY 1), | |
| 130 | + oct10q8k AS (SELECT {grp} g, count(DISTINCT gkey) projects, count(DISTINCT cik) issuers, count(*) mentions FROM k2 WHERE run='october' AND form_type IN ('10-Q','8-K') GROUP BY 1), | |
| 131 | + ledr AS (SELECT {grp} g, count(*) ledger_rows, count(*) FILTER (WHERE run='october') october_rows, count(DISTINCT cik) issuers_ledger FROM ledm GROUP BY 1) | |
| 132 | +SELECT a.g cohort, a.issuers, a.projects projects_all, round(a.projects*1.0/a.issuers,1) per_issuer_all, | |
| 133 | + o.projects projects_october_only, round(o.projects*1.0/o.issuers,1) per_issuer_october_only, | |
| 134 | + q.projects projects_oct_10q_8k, round(q.projects*1.0/q.issuers,1) per_issuer_oct_10q_8k, | |
| 135 | + l.ledger_rows, l.october_rows, round(l.ledger_rows*1.0/l.issuers_ledger,1) ledger_rows_per_issuer, round(l.october_rows*1.0/l.issuers_ledger,1) october_rows_per_issuer | |
| 136 | +FROM allm a JOIN octm o USING (g) JOIN oct10q8k q USING (g) JOIN ledr l USING (g) ORDER BY 1""") | |
| 137 | +iss = con.execute(f""" | |
| 138 | +WITH p AS (SELECT cik, count(DISTINCT gkey) projects, count(DISTINCT gkey) FILTER (WHERE run='october') projects_oct FROM k2 GROUP BY 1), | |
| 139 | + l AS (SELECT cik, any_value(in_sp500) sp, count(*) rows, count(*) FILTER (WHERE run='october') rows_oct, count(DISTINCT accession_number) filings FROM ledm GROUP BY 1) | |
| 140 | +SELECT l.cik, l.sp::int, coalesce(p.projects,0), coalesce(p.projects_oct,0), l.rows, l.rows_oct, l.filings FROM l LEFT JOIN p USING (cik)""").fetchall() | |
| 141 | +A = np.array([[x[1], x[2], x[3], x[4], x[5], x[6]] for x in iss], dtype=float) | |
| 142 | +def ols(y, X, names): | |
| 143 | + X = np.column_stack([np.ones(len(y)), X]); b, *_ = np.linalg.lstsq(X, y, rcond=None); e = y - X @ b | |
| 144 | + XtX_inv = np.linalg.inv(X.T @ X); meat = (X * (e**2)[:, None]).T @ X; V = XtX_inv @ meat @ XtX_inv * len(y)/(len(y)-X.shape[1]) | |
| 145 | + se = np.sqrt(np.diag(V)); return {n: {"coef": float(bb), "hc1_se": float(s)} for n, bb, s in zip(["const"]+names, b, se)} | {"n": int(len(y)), "r2": float(1 - (e**2).sum()/((y-y.mean())**2).sum())} | |
| 146 | +REG = {} | |
| 147 | +REG["projects_all_on_sp500"] = ols(A[:,1], A[:,[0]], ["in_sp500"]) | |
| 148 | +REG["projects_october_only_on_sp500"] = ols(A[:,2], A[:,[0]], ["in_sp500"]) | |
| 149 | +REG["projects_all_on_sp500_log_rows"] = ols(A[:,1], np.column_stack([A[:,0], np.log1p(A[:,3])]), ["in_sp500", "log1p_ledger_rows"]) | |
| 150 | +REG["projects_october_only_on_sp500_log_rows_oct"] = ols(A[:,2], np.column_stack([A[:,0], np.log1p(A[:,4])]), ["in_sp500", "log1p_october_rows"]) | |
| 151 | +REG["log_projects_all_on_sp500_log_rows"] = ols(np.log1p(A[:,1]), np.column_stack([A[:,0], np.log1p(A[:,3])]), ["in_sp500", "log1p_ledger_rows"]) | |
| 152 | +REG["log_projects_oct_on_sp500_log_rows_oct"] = ols(np.log1p(A[:,2]), np.column_stack([A[:,0], np.log1p(A[:,4])]), ["in_sp500", "log1p_october_rows"]) | |
| 153 | +REG["note"] = "issuer-level OLS, HC1 standard errors; 'october only' counts projects whose key is touched by at least one October-run mention; in_sp500 = cohort dummy (S&P 500 issuers were extracted in two runs and with two 10-K segmenters)" | |
| 154 | +r.write_json("table6_cohort_regression.json", REG) | |
| 155 | + | |
| 156 | +# ---- (g) fiscal_year audit and derived fill ------------------------------------------------------------------ | |
| 157 | +G = {} | |
| 158 | +G["mentions_fiscal_year_null_by_run_form"] = con.execute("select run, form_type, count(*) filter (where fiscal_year is null), count(*) from pm group by 1,2 order by 2,1").fetchall() | |
| 159 | +G["src_filings_fiscal_year_null_by_form"] = con.execute("select form_type, count(*) filter (where fiscal_year is null), count(*) filter (where report_date is null), count(*) from src_filings group by 1").fetchall() | |
| 160 | +G["null_mentions_fillable_from_src_filings"] = con.execute("""select count(*) filter (where f.fiscal_year is not null), count(*) filter (where f.fiscal_year is null and f.report_date is not null), count(*) | |
| 161 | + from pm m join src_filings f using(accession_number) where m.fiscal_year is null""").fetchone() | |
| 162 | +G["ledger_rows_filing_fiscal_year_null_by_run"] = con.execute("select run, count(*) filter (where filing_fiscal_year is null), count(*) from ledm group by 1").fetchall() | |
| 163 | +G["fill_rule"] = "fiscal_year_fill = coalesce(src_filings.fiscal_year, year(report_date), year(filing_date)); derived, not stored" | |
| 164 | +r.dump("fiscal_year_fill_by_accession.csv", con, """ | |
| 165 | +SELECT DISTINCT m.accession_number, m.form_type, f.filing_date, f.report_date, f.fiscal_year AS fiscal_year_src, | |
| 166 | + coalesce(f.fiscal_year, year(f.report_date), year(f.filing_date)) AS fiscal_year_fill, | |
| 167 | + CASE WHEN f.fiscal_year IS NOT NULL THEN 'src_filings' WHEN f.report_date IS NOT NULL THEN 'year(report_date)' ELSE 'year(filing_date)' END AS fill_source | |
| 168 | +FROM pm m JOIN src_filings f USING (accession_number) WHERE m.fiscal_year IS NULL ORDER BY 1""") | |
| 169 | +G["mentions_null_by_fill_source"] = con.execute("""select case when f.fiscal_year is not null then 'src_filings' when f.report_date is not null then 'year(report_date)' else 'year(filing_date)' end s, count(*) | |
| 170 | + from pm m join src_filings f using(accession_number) where m.fiscal_year is null group by 1""").fetchall() | |
| 171 | +G["mentions_fiscal_year_vs_filing_fiscal_year_mismatch_nonnull"] = con.execute("""select count(*) from pm m join src_filings f using(accession_number) where m.fiscal_year is not null and f.fiscal_year is not null and m.fiscal_year <> f.fiscal_year""").fetchone()[0] | |
| 172 | +r.write_json("fiscal_year_audit.json", G) | |
| 173 | + | |
| 174 | +# ---- (h) 10-K funnel ---------------------------------------------------------------------------------------- | |
| 175 | +H = {} | |
| 176 | +H["parsed_10k_rows_total"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K'").fetchone()[0] | |
| 177 | +H["parsed_10k_rows_out_of_scope_item"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name not in ('business','risk_factors','properties','md_and_a','quantitative_disclosures')").fetchone()[0] | |
| 178 | +H["parsed_10k_rows_in_scope"] = H["parsed_10k_rows_total"] - H["parsed_10k_rows_out_of_scope_item"] | |
| 179 | +H["parsed_10k_in_scope_rows_of_legacy_accessions"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name in ('business','risk_factors','properties','md_and_a','quantitative_disclosures') and s.accession_number in (select accession_number from spid_sections)").fetchone()[0] | |
| 180 | +H["parsed_10k_in_scope_rows_lt30_words_non_legacy"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name in ('business','risk_factors','properties','md_and_a','quantitative_disclosures') and s.accession_number not in (select accession_number from spid_sections) and s.word_count < 30").fetchone()[0] | |
| 181 | +H["parsed_10k_in_scope_rows_ge30_non_legacy"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name in ('business','risk_factors','properties','md_and_a','quantitative_disclosures') and s.accession_number not in (select accession_number from spid_sections) and s.word_count >= 30").fetchone()[0] | |
| 182 | +H["parsed_10k_in_scope_rows_ge30_non_legacy_in_ledger"] = con.execute("select count(*) from src_sections s join src_filings f using(accession_number) where f.form_type='10-K' and s.section_name in ('business','risk_factors','properties','md_and_a','quantitative_disclosures') and s.accession_number not in (select accession_number from spid_sections) and s.word_count >= 30 and s.section_id in (select section_id from extracted_sections)").fetchone()[0] | |
| 183 | +H["legacy_rows"] = con.execute("select count(*) from spid_sections").fetchone()[0] | |
| 184 | +H["legacy_rows_in_ledger"] = con.execute("select count(*) from spid_sections where section_id in (select section_id from extracted_sections)").fetchone()[0] | |
| 185 | +H["legacy_rows_lt30"] = con.execute("select count(*) from spid_sections where word_count<30").fetchone()[0] | |
| 186 | +H["ledger_10k_rows"] = con.execute("select count(*) from ledm where form_type='10-K'").fetchone()[0] | |
| 187 | +H["ledger_10k_by_segmenter"] = con.execute("select segmenter, count(*) from ledm where form_type='10-K' group by 1").fetchall() | |
| 188 | +H["funnel"] = f"{H['parsed_10k_rows_total']} parsed 10-K rows - {H['parsed_10k_rows_out_of_scope_item']} out-of-scope items - {H['parsed_10k_in_scope_rows_of_legacy_accessions']} rows of legacy-ingested filings - {H['parsed_10k_in_scope_rows_lt30_words_non_legacy']} rows < 30 words = {H['parsed_10k_in_scope_rows_ge30_non_legacy']} (ledgered: {H['parsed_10k_in_scope_rows_ge30_non_legacy_in_ledger']}); + {H['legacy_rows_in_ledger']} legacy rows = {H['parsed_10k_in_scope_rows_ge30_non_legacy_in_ledger'] + H['legacy_rows_in_ledger']} vs ledger 10-K rows {H['ledger_10k_rows']}" | |
| 189 | +H["expected_ledger_10k"] = 143704 | |
| 190 | +# 10-Q items in scope (all seven submitted?) | |
| 191 | +H["ledger_10q_items"] = con.execute("select section_name, count(*) from ledm where form_type='10-Q' group by 1 order by 2 desc").fetchall() | |
| 192 | +H["parsed_10q_items"] = con.execute("select s.section_name, count(*), count(*) filter (where s.word_count<30) from src_sections s join src_filings f using(accession_number) where f.form_type='10-Q' group by 1 order by 2 desc").fetchall() | |
| 193 | +H["ledger_8k_items"] = con.execute("select section_name, count(*) from ledm where form_type='8-K' group by 1 order by 2 desc").fetchall() | |
| 194 | +r.write_json("funnel_10k.json", H) | |
| 195 | +r.dump("legacy_vs_parsed_sp500_items.csv", con, """ | |
| 196 | +SELECT 'legacy_ingest' segmenter, section_name, count(*) n, round(median(word_count)) median_words, count(*) FILTER (WHERE word_count<30) lt30 FROM spid_sections GROUP BY 1,2 | |
| 197 | +UNION ALL | |
| 198 | +SELECT 'parsed_sp500_10k' segmenter, s.section_name, count(*), round(median(s.word_count)), count(*) FILTER (WHERE s.word_count<30) FROM src_sections s JOIN src_filings f USING (accession_number) JOIN src_companies co ON f.cik=co.cik WHERE f.form_type='10-K' AND co.in_sp500 GROUP BY 1,2 | |
| 199 | +UNION ALL | |
| 200 | +SELECT 'parsed_nonsp500_10k' segmenter, s.section_name, count(*), round(median(s.word_count)), count(*) FILTER (WHERE s.word_count<30) FROM src_sections s JOIN src_filings f USING (accession_number) JOIN src_companies co ON f.cik=co.cik WHERE f.form_type='10-K' AND NOT co.in_sp500 GROUP BY 1,2 | |
| 201 | +ORDER BY 1,3 DESC""") | |
| 202 | +r.finish() | |
added
results/runs/fiscal_year_audit.json
+85 −0
@@ -0,0 +1,85 @@ | ||
| 1 | +{ | |
| 2 | + "mentions_fiscal_year_null_by_run_form": [ | |
| 3 | + [ | |
| 4 | + "backfilled_june", | |
| 5 | + "10-K", | |
| 6 | + 15787, | |
| 7 | + 15787 | |
| 8 | + ], | |
| 9 | + [ | |
| 10 | + "october", | |
| 11 | + "10-K", | |
| 12 | + 823, | |
| 13 | + 65708 | |
| 14 | + ], | |
| 15 | + [ | |
| 16 | + "backfilled_june", | |
| 17 | + "10-Q", | |
| 18 | + 5817, | |
| 19 | + 5817 | |
| 20 | + ], | |
| 21 | + [ | |
| 22 | + "october", | |
| 23 | + "10-Q", | |
| 24 | + 28074, | |
| 25 | + 146187 | |
| 26 | + ], | |
| 27 | + [ | |
| 28 | + "backfilled_june", | |
| 29 | + "8-K", | |
| 30 | + 18326, | |
| 31 | + 18326 | |
| 32 | + ], | |
| 33 | + [ | |
| 34 | + "october", | |
| 35 | + "8-K", | |
| 36 | + 0, | |
| 37 | + 56240 | |
| 38 | + ] | |
| 39 | + ], | |
| 40 | + "src_filings_fiscal_year_null_by_form": [ | |
| 41 | + [ | |
| 42 | + "10-Q", | |
| 43 | + 22542, | |
| 44 | + 22542, | |
| 45 | + 95167 | |
| 46 | + ], | |
| 47 | + [ | |
| 48 | + "8-K", | |
| 49 | + 103927, | |
| 50 | + 103928, | |
| 51 | + 400606 | |
| 52 | + ], | |
| 53 | + [ | |
| 54 | + "10-K", | |
| 55 | + 7655, | |
| 56 | + 7655, | |
| 57 | + 31775 | |
| 58 | + ] | |
| 59 | + ], | |
| 60 | + "null_mentions_fillable_from_src_filings": [ | |
| 61 | + 0, | |
| 62 | + 0, | |
| 63 | + 68827 | |
| 64 | + ], | |
| 65 | + "ledger_rows_filing_fiscal_year_null_by_run": [ | |
| 66 | + [ | |
| 67 | + "backfilled_june", | |
| 68 | + 154964, | |
| 69 | + 154964 | |
| 70 | + ], | |
| 71 | + [ | |
| 72 | + "october", | |
| 73 | + 114051, | |
| 74 | + 968134 | |
| 75 | + ] | |
| 76 | + ], | |
| 77 | + "fill_rule": "fiscal_year_fill = coalesce(src_filings.fiscal_year, year(report_date), year(filing_date)); derived, not stored", | |
| 78 | + "mentions_null_by_fill_source": [ | |
| 79 | + [ | |
| 80 | + "year(filing_date)", | |
| 81 | + 68827 | |
| 82 | + ] | |
| 83 | + ], | |
| 84 | + "mentions_fiscal_year_vs_filing_fiscal_year_mismatch_nonnull": 0 | |
| 85 | +} | |
| \ No newline at end of file | ||
added
results/runs/fiscal_year_fill_by_accession.csv
+19639 −0
Large diff (19,639 lines) is not loaded by default to keep the page fast.Load diff
Diff truncated — file too large.