Repository Public Research preview
1"""n1_04_validation_samples.py — human-validation sampling frames (AI_C-07, DATA-11, THEO-14, EDIT-05).23Chunk composite = mean of corpus-standardized score_ai_adoption and score_ai_infrastructure (chunk-level analogue of4the paper's AI Compos). Deterministic selection by hash(chunk_id || seed), seed 20261105. Text clipped to 2,000 chars.5Frames:6 A stratified : cohorts 2012 and 2024 x within-year composite decile x 20 chunks = 400 (blind coding sheet + key)7 B top_firms : 20 top-composite chunks of each of the 30 top firms of Table 4 (>= 5 years), IDA and HE flagged8 C median_firms: 20 top-composite chunks of each of the 30 firms around the median of the same ranking9 D top_by_year: 25 top-composite chunks of 2010, 2014, 2018, 202210Outputs: results/validation_*.csv, validation_frames_summary.json, validation_codebook.md11"""12import os, sys, json13import numpy as np, pandas as pd14sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))15from n1_common import * # noqa1617c = connect(); views(c)18HERE = os.path.dirname(os.path.abspath(__file__))19lists = json.load(open(f"{HERE}/firm_lists_t4.json"))20top30 = [r["ticker"] for r in lists["top30"]]; med30 = [r["ticker"] for r in lists["median30"]]21kw_path = f"{RES}/chunk_kw.parquet"22has_kw = os.path.exists(kw_path)23if has_kw:24 c.execute(f"CREATE OR REPLACE VIEW chunk_kw AS SELECT * FROM read_parquet('{kw_path}')")25else:26 c.execute("CREATE OR REPLACE VIEW chunk_kw AS SELECT chunk_id, NULL::INT n_core, NULL::INT n_ext FROM chunk_scores LIMIT 0")2728with Timer("base table"):29 mu_a, sd_a, mu_i, sd_i = c.execute("SELECT avg(score_ai_adoption), stddev_pop(score_ai_adoption), avg(score_ai_infrastructure), stddev_pop(score_ai_infrastructure) FROM chunk_scores").fetchone()30 c.execute(f"""CREATE OR REPLACE TABLE base AS31 SELECT cs.chunk_id, cs.accession_number, cs.cik, cs.ticker, cs.form_type, cs.filing_date, cs.fiscal_year, cs.section_name,32 cs.score_ai_adoption, cs.score_ai_infrastructure, cs.score_ai_disruption_risk, cs.score_cybersecurity, cs.score_automation_labor,33 ((cs.score_ai_adoption - {mu_a})/{sd_a} + (cs.score_ai_infrastructure - {mu_i})/{sd_i})/2 AS chunk_composite,34 k.n_core, k.n_ext, hash(cs.chunk_id || '{SEED}') AS h35 FROM chunk_scores cs LEFT JOIN chunk_kw k USING (chunk_id)36 WHERE cs.fiscal_year BETWEEN 2010 AND 2024""")37 n_base = c.execute("SELECT count(*) FROM base").fetchone()[0]3839with Timer("frames"):40 c.execute("""CREATE OR REPLACE TABLE fa AS41 SELECT *, 'A_stratified' AS frame, fiscal_year::VARCHAR || '-D' || lpad(decile::VARCHAR, 2, '0') AS stratum FROM (42 SELECT *, ntile(10) OVER (PARTITION BY fiscal_year ORDER BY chunk_composite) AS decile FROM base WHERE fiscal_year IN (2012, 2024))43 QUALIFY row_number() OVER (PARTITION BY fiscal_year, decile ORDER BY h) <= 20""")44 cut = c.execute("""SELECT fiscal_year, decile, min(chunk_composite) lo, max(chunk_composite) hi, count(*) n FROM (45 SELECT fiscal_year, chunk_composite, ntile(10) OVER (PARTITION BY fiscal_year ORDER BY chunk_composite) AS decile FROM base WHERE fiscal_year IN (2012, 2024))46 GROUP BY 1,2 ORDER BY 1,2""").df()47 tl = ",".join(f"'{t}'" for t in top30); ml = ",".join(f"'{t}'" for t in med30)48 c.execute(f"""CREATE OR REPLACE TABLE fb AS SELECT *, 0 AS decile, 'B_top_firms' AS frame, ticker AS stratum FROM base WHERE ticker IN ({tl})49 QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY chunk_composite DESC, h) <= 20""")50 c.execute(f"""CREATE OR REPLACE TABLE fc AS SELECT *, 0 AS decile, 'C_median_firms' AS frame, ticker AS stratum FROM base WHERE ticker IN ({ml})51 QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY chunk_composite DESC, h) <= 20""")52 c.execute("""CREATE OR REPLACE TABLE fd AS SELECT *, 0 AS decile, 'D_top_by_year' AS frame, fiscal_year::VARCHAR AS stratum FROM base WHERE fiscal_year IN (2010, 2014, 2018, 2022)53 QUALIFY row_number() OVER (PARTITION BY fiscal_year ORDER BY chunk_composite DESC, h) <= 25""")54 sel = c.execute("""SELECT s.*, ch.chunk_order, ch.word_count, ch.char_count, ch.chunk_text, co.company_name, co.sector, co.sub_industry55 FROM (SELECT * FROM fa UNION ALL SELECT * FROM fb UNION ALL SELECT * FROM fc UNION ALL SELECT * FROM fd) s56 JOIN chunks ch USING (chunk_id) LEFT JOIN companies co ON co.cik = s.cik""").df()5758sel["text"] = sel["chunk_text"].str.slice(0, 2000); sel["text_truncated_2000"] = sel["char_count"] > 200059sel = sel.drop(columns=["chunk_text", "h"])60sel["special"] = np.where(sel.ticker.isin(["IDA", "HE"]), "utility_in_top12", "")61sel = sel.sort_values(["frame", "stratum", "chunk_composite"], ascending=[True, True, False]).reset_index(drop=True)62sel.insert(0, "sample_id", [f"N1-{i+1:05d}" for i in range(len(sel))])63for col in ["label_ai_substantive", "label_ai_mention", "label_boilerplate", "label_topic", "reader", "notes"]:64 sel[col] = ""65meta_cols = ["sample_id", "frame", "stratum", "decile", "chunk_id", "accession_number", "cik", "ticker", "company_name", "sector", "sub_industry", "form_type",66 "filing_date", "fiscal_year", "section_name", "chunk_order", "word_count", "char_count", "text_truncated_2000",67 "score_ai_adoption", "score_ai_infrastructure", "score_ai_disruption_risk", "score_cybersecurity", "score_automation_labor",68 "chunk_composite", "n_core", "n_ext", "special"]69label_cols = ["label_ai_substantive", "label_ai_mention", "label_boilerplate", "label_topic", "reader", "notes"]70for fr, fname in [("A_stratified", "validation_stratified_key.csv"), ("B_top_firms", "validation_top_firms.csv"),71 ("C_median_firms", "validation_median_firms.csv"), ("D_top_by_year", "validation_top_by_year.csv")]:72 sel[sel.frame == fr][meta_cols + ["text"] + label_cols].to_csv(f"{RES}/{fname}", index=False, float_format="%.5f")73# blind coding sheet: stratified + top-by-year, shuffled, without scores / decile / keyword counts74rng = np.random.default_rng(SEED)75blind = sel[sel.frame.isin(["A_stratified", "D_top_by_year"])].copy()76blind = blind.iloc[rng.permutation(len(blind))]77blind[["sample_id", "form_type", "fiscal_year", "section_name", "sector", "text"] + label_cols].to_csv(f"{RES}/validation_coding_sheet_blind.csv", index=False)7879summ = {"seed": SEED, "n_base_chunks_2010_2024": int(n_base), "composite_moments": {"mu_adopt": mu_a, "sd_adopt": sd_a, "mu_infra": mu_i, "sd_infra": sd_i},80 "frames": sel.groupby("frame").agg(n=("sample_id", "count"), strata=("stratum", "nunique"), mean_composite=("chunk_composite", "mean"),81 share_core_hit=("n_core", lambda s: float((s.fillna(0) > 0).mean())),82 share_text_truncated=("text_truncated_2000", "mean")).round(4).to_dict("index"),83 "stratified_by_decile": sel[sel.frame == "A_stratified"].groupby(["fiscal_year", "decile"]).agg(n=("sample_id", "count"), mean_composite=("chunk_composite", "mean"),84 mean_adopt=("score_ai_adoption", "mean"), share_core_hit=("n_core", lambda s: float((s.fillna(0) > 0).mean())),85 share_md_and_a=("section_name", lambda s: float((s == "md_and_a").mean())),86 share_financial_statements=("section_name", lambda s: float((s == "financial_statements").mean()))).round(4).reset_index().to_dict("records"),87 "decile_cutpoints": cut.round(4).to_dict("records"),88 "top_firms_frame_by_firm": sel[sel.frame == "B_top_firms"].groupby(["ticker", "sector"]).agg(n=("sample_id", "count"), mean_composite=("chunk_composite", "mean"),89 share_core_hit=("n_core", lambda s: float((s.fillna(0) > 0).mean())), top_section=("section_name", lambda s: s.mode().iloc[0]),90 share_8k=("form_type", lambda s: float((s == "8-K").mean()))).round(4).reset_index().to_dict("records"),91 "median_firms_frame_by_firm": sel[sel.frame == "C_median_firms"].groupby(["ticker", "sector"]).agg(n=("sample_id", "count"), mean_composite=("chunk_composite", "mean"),92 share_core_hit=("n_core", lambda s: float((s.fillna(0) > 0).mean()))).round(4).reset_index().to_dict("records"),93 "top_by_year_frame": sel[sel.frame == "D_top_by_year"].groupby("fiscal_year").agg(n=("sample_id", "count"), mean_composite=("chunk_composite", "mean"),94 share_core_hit=("n_core", lambda s: float((s.fillna(0) > 0).mean())), firms=("ticker", "nunique"),95 top_sector=("sector", lambda s: s.mode().iloc[0])).round(4).reset_index().to_dict("records"),96 "ida_he_top_chunks": sel[(sel.frame == "B_top_firms") & sel.ticker.isin(["IDA", "HE"])][["sample_id", "ticker", "fiscal_year", "form_type", "section_name", "chunk_composite", "n_core"]].round(4).to_dict("records")}97dump_json(summ, f"{RES}/validation_frames_summary.json")9899codebook = """# Codebook — FIRC-0002 N1 human-validation frames (seed 20261105)100101Files: `validation_coding_sheet_blind.csv` (A stratified 400 + D top-by-year 100, shuffled, no scores shown — code this one),102`validation_stratified_key.csv` (same A rows with decile, scores and keyword counts — for the analyst, not the coders),103`validation_top_firms.csv` (B: 20 top chunks x 30 top firms of Table 4, IDA / HE flagged in `special`),104`validation_median_firms.csv` (C: 20 top chunks x 30 median firms), `validation_top_by_year.csv` (D with scores).105106Columns to fill (two readers, independently; kappa computed on `label_ai_substantive`):107- `label_ai_substantive`: 1 if the passage describes the FIRM'S OWN adoption, deployment, investment, product or108 infrastructure in artificial intelligence / machine learning (including generative AI); 0 otherwise. Generic risk-factor109 boilerplate, third-party / customer AI, pure automation without AI, cybersecurity without AI, generic data centres or110 cloud capacity without AI purpose are 0.111- `label_ai_mention`: 1 if AI / machine learning is mentioned at all (any register), 0 otherwise.112- `label_boilerplate`: 1 if the passage is standardized language repeated across filings (risk factors, safe-harbor113 statements, exhibit lists, accounting policies), 0 if firm-specific narrative.114- `label_topic`: one of AI | automation | cybersecurity | data_center_or_cloud | IT_generic | analytics_or_data |115 utility_load_or_energy | financial_boilerplate | other.116- `reader`: initials. `notes`: free text.117118Precision by decile = mean of `label_ai_substantive` within each 2012 / 2024 decile (join on `sample_id` to the key file);119precision by sector from the `sector` column; the B/C frames give the top-firm precision (IDA / HE in particular).120Text is clipped at 2,000 characters (`text_truncated_2000` in the key files); the full chunk is in `filing_chunks` by `chunk_id`.121"""122open(f"{RES}/validation_codebook.md", "w").write(codebook)123print(json.dumps(summ["frames"], indent=1)); print(len(sel), "rows")124