1"""r3k_23_construct_validity.py — Stage D (FIRC-0002, package N1 laptop part / L7, cluster J: EDIT-05, AI_C-07, DATA-11, THEO-14, ADVE-03, AI_C-17).2Construct-validity rows for Tables 5 / 7 from the node audit outputs (results/n1/, N1 executor on M4M64a): (i) the keyword-count AI3baseline (published AI lexicon regex over the section texts: kw_core_wmean = core-term hits per 1,000 words, section-weighted) and4(ii) the alternative-anchor composite under AR-FIRC0002-2 (anchors stripped of cybersecurity / automation terms, scored against the5STORED chunk embeddings). Each signal is standardized within year, sorted into quintiles (raw and winsorized forward returns) and6entered in a Fama-MacBeth regression; rank correlations with the baseline composite are reported. Also the Utilities / Energy7exclusion bound for Tables 5 / 9. Writes results/r3k_v2/results_construct.json and paper/tables/tab_construct_validity.tex.8Run: .venv-review/bin/python -I scripts/r3k_23_construct_validity.py9"""10import os, sys, json, numpy as np, pandas as pd, statsmodels.api as sm11sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import r3k_inference as I12V = "data/processed_r3k_v2"; T = "results/r3k_v2"; P = "paper/tables"; PRE, POST, ALL = range(2010, 2022), range(2022, 2025), range(2010, 2025)13df = pd.read_parquet(f"{V}/analysis_panel_v2.parquet"); RES = {"script": "r3k_23_construct_validity.py", "inputs": []}14def zc(g): return (g - g.mean()) / g.std(ddof=0)15kw = pd.read_csv("results/n1/keyword_firm_year.csv", dtype={"cik": str}) if os.path.exists("results/n1/keyword_firm_year.csv") else None16alt = pd.read_csv("results/n1/alt_firm_year.csv", dtype={"cik": str}) if os.path.exists("results/n1/alt_firm_year.csv") else None17if kw is not None:18 kw["cik"] = kw.cik.str.zfill(10); df = df.merge(kw[["cik", "fiscal_year", "kw_core_wmean", "kw_core_mean", "any_core_share", "kw_core_10k"]], on=["cik", "fiscal_year"], how="left"); RES["inputs"].append("results/n1/keyword_firm_year.csv")19 df["z_KW"] = df.groupby("fiscal_year")["kw_core_wmean"].transform(zc); df["z_KWlog"] = df.groupby("fiscal_year")["kw_core_wmean"].transform(lambda s: zc(np.log1p(s)))20if alt is not None:21 alt["cik"] = alt.cik.str.zfill(10); df = df.merge(alt[["cik", "fiscal_year", "alt_ai_adoption_strip_wmean", "alt_ai_infrastructure_strip_wmean", "alt_ai_adoption_core_wmean", "alt_cybersecurity_strip_wmean", "alt_automation_labor_strip_wmean"]], on=["cik", "fiscal_year"], how="left"); RES["inputs"].append("results/n1/alt_firm_year.csv")22 df["z_ALT"] = (df.groupby("fiscal_year")["alt_ai_adoption_strip_wmean"].transform(zc) + df.groupby("fiscal_year")["alt_ai_infrastructure_strip_wmean"].transform(zc)) / 2; df["z_ALT"] = df.groupby("fiscal_year")["z_ALT"].transform(zc)23 df["z_ALTcore"] = df.groupby("fiscal_year")["alt_ai_adoption_core_wmean"].transform(zc); df["z_CYBERstrip"] = df.groupby("fiscal_year")["alt_cybersecurity_strip_wmean"].transform(zc)24def sort_table(d, z, oc, yrs=ALL, minn=40):25 ls = {}26 for yr, g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"):27 g = g.dropna(subset=[z, oc]).copy()28 if len(g) < minn: continue29 g["q"] = pd.qcut(g[z].rank(method="first"), 5, labels=False) + 1; ls[int(yr)] = float(g[g.q == 5][oc].mean() - g[g.q == 1][oc].mean())30 v = list(ls.values()); mu, t = I.nw_t(v); return {"mean": round(mu, 4) if v else None, "t": round(t, 2) if t == t else None, "t_plain": round(I.plain_t(v), 2) if len(v) > 1 else None, "ci95": [round(x, 4) for x in I.ci95(v)] if len(v) > 1 else None, "n": len(v), "by_cohort": {k: round(x, 4) for k, x in ls.items()}}31def fm(d, z, oc):32 c = []33 for yr, g in d.groupby("fiscal_year"):34 g = g.dropna(subset=[oc, z, "z_size", "z_mom", "z_vol"])35 if len(g) < 40: continue36 c.append(sm.OLS(g[oc], sm.add_constant(g[[z, "z_size", "z_mom", "z_vol"]])).fit().params[z])37 mu, t = I.nw_t(c); return {"slope_pct": round(mu * 100, 2), "t": round(t, 2), "t_plain": round(I.plain_t(c), 2), "ci95_pct": [round(x * 100, 2) for x in I.ci95(c)], "n_years": len(c)}38SIG = [("z_AI_COMPOS", "AI composite (baseline)")] + ([("z_KW", "Keyword baseline (core AI terms per 1,000 words)"), ("z_KWlog", "Keyword baseline, log(1 + x)")] if kw is not None else []) + ([("z_ALT", "Stripped-anchor composite (AR-2)"), ("z_ALTcore", "Core-AI adoption anchor (AR-2)"), ("z_CYBERstrip", "Cybersecurity anchor stripped of AI terms (AR-2)")] if alt is not None else [])39RES["signals"] = {}40for z, l in SIG:41 d = df.dropna(subset=[z]); RES["signals"][z] = {"label": l, "n": int(len(d)), "spearman_with_composite": round(float(d[[z, "z_AI_COMPOS"]].corr(method="spearman").iloc[0, 1]), 3), "share_zero": round(float((d[z.replace("z_KW", "kw_core_wmean")] == 0).mean()), 3) if z == "z_KW" else None,42 "full": sort_table(d, z, "fwd_ret"), "full_w": sort_table(d, z, "fwd_w"), "pre_w": sort_table(d, z, "fwd_w", PRE), "post_w": sort_table(d, z, "fwd_w", POST), "fm": fm(d, z, "fwd_ret"), "fm_w": fm(d, z, "fwd_w")}43 print(l, "| rho", RES["signals"][z]["spearman_with_composite"], "| full w", RES["signals"][z]["full_w"]["mean"], RES["signals"][z]["full_w"]["t"], "| FM w", RES["signals"][z]["fm_w"], flush=True)44# horse race composite vs keyword / stripped anchors (FM, winsorized)45def fm_multi(d, zs, oc="fwd_w"):46 c = {z: [] for z in zs}47 for yr, g in d.groupby("fiscal_year"):48 g = g.dropna(subset=[oc] + zs + ["z_size", "z_mom", "z_vol"])49 if len(g) < 40: continue50 r = sm.OLS(g[oc], sm.add_constant(g[zs + ["z_size", "z_mom", "z_vol"]])).fit()51 for z in zs: c[z].append(r.params[z])52 return {z: {"slope_pct": round(I.nw_t(v)[0] * 100, 2), "t": round(I.nw_t(v)[1], 2), "t_plain": round(I.plain_t(v), 2)} for z, v in c.items()}53RES["horse_race_w"] = {}54if kw is not None: RES["horse_race_w"]["composite_vs_keyword"] = fm_multi(df, ["z_AI_COMPOS", "z_KW"])55if alt is not None: RES["horse_race_w"]["composite_vs_stripped"] = fm_multi(df, ["z_AI_COMPOS", "z_ALT"])56# Utilities / Energy exclusion bound57ex = df[~df.sector.isin(["Utilities", "Energy"])]; RES["ex_utilities_energy"] = {"n": int(len(ex)), "full": sort_table(ex, "z_AI_COMPOS", "fwd_ret"), "full_w": sort_table(ex, "z_AI_COMPOS", "fwd_w"), "pre_w": sort_table(ex, "z_AI_COMPOS", "fwd_w", PRE), "post_w": sort_table(ex, "z_AI_COMPOS", "fwd_w", POST), "fm_w": fm(ex, "z_AI_COMPOS", "fwd_w"), "fm": fm(ex, "z_AI_COMPOS", "fwd_ret")}58# top-of-ranking sector composition (IDA / HE question)59top = df[df.groupby("fiscal_year")["z_AI_COMPOS"].transform(lambda s: s >= s.quantile(.8))]; RES["q5_sector_shares"] = top.sector.value_counts(normalize=True).round(3).head(8).to_dict(); RES["utilities_share_q5_vs_all"] = [round(float((top.sector == "Utilities").mean()), 3), round(float((df.sector == "Utilities").mean()), 3)]60json.dump(RES, open(f"{T}/results_construct.json", "w"), indent=2, default=str)61def pp(x): return "--" if x is None else f"{x * 100:+.1f}"62def tv(t): return "--" if t is None or (isinstance(t, float) and np.isnan(t)) else f"{t:.2f}"63body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nSignal & $N$ & $\\rho$ with composite & Full Q5$-$Q1 raw & Full Q5$-$Q1 wins. (NW $t$ / plain $t$) & Pre wins. & Post wins. & FM slope raw (NW $t$) & FM slope wins. (NW $t$) \\\\\n\\midrule\n"64for z, l in SIG:65 s = RES["signals"][z]; body += f"{l} & {s['n']:,} & {s['spearman_with_composite']:.2f} & {pp(s['full']['mean'])} & {pp(s['full_w']['mean'])} ({tv(s['full_w']['t'])} / {tv(s['full_w']['t_plain'])}) & {pp(s['pre_w']['mean'])} & {pp(s['post_w']['mean'])} & {s['fm']['slope_pct']:+.2f} ({tv(s['fm']['t'])}) & {s['fm_w']['slope_pct']:+.2f} ({tv(s['fm_w']['t'])}) \\\\\n".replace(",", "{,}", 1) if False else f"{l} & {s['n']} & {s['spearman_with_composite']:.2f} & {pp(s['full']['mean'])} & {pp(s['full_w']['mean'])} ({tv(s['full_w']['t'])} / {tv(s['full_w']['t_plain'])}) & {pp(s['pre_w']['mean'])} & {pp(s['post_w']['mean'])} & {s['fm']['slope_pct']:+.2f} ({tv(s['fm']['t'])}) & {s['fm_w']['slope_pct']:+.2f} ({tv(s['fm_w']['t'])}) \\\\\n"66e = RES["ex_utilities_energy"]; body += f"AI composite, excluding Utilities and Energy & {e['n']} & & {pp(e['full']['mean'])} & {pp(e['full_w']['mean'])} ({tv(e['full_w']['t'])} / {tv(e['full_w']['t_plain'])}) & {pp(e['pre_w']['mean'])} & {pp(e['post_w']['mean'])} & {e['fm']['slope_pct']:+.2f} ({tv(e['fm']['t'])}) & {e['fm_w']['slope_pct']:+.2f} ({tv(e['fm_w']['t'])}) \\\\\n\\midrule\n"67for k, v in RES["horse_race_w"].items(): body += f"Horse race (FM, wins.): {k.replace('_', ' ')} & \\multicolumn{{8}}{{l}}{{" + "; ".join(f"{z}: {x['slope_pct']:+.2f} ({x['t']:.2f})" for z, x in v.items()) + "} \\\\\n"68note = f"Keyword baseline from the node audit (code/n1, published AI lexicon over the section texts; share of firm-years with zero core hits {RES['signals'].get('z_KW', {}).get('share_zero', '--')}); stripped-anchor composite = anchors without the cybersecurity / automation vocabulary, scored against the stored chunk embeddings (AR-FIRC0002-2, no corpus re-embedding). Utilities are {RES['utilities_share_q5_vs_all'][0] * 100:.1f}\\% of top-quintile firm-years vs {RES['utilities_share_q5_vs_all'][1] * 100:.1f}\\% of the sample. NW $t$ over cohorts (3 lags); FM controls size, momentum, volatility."69open(f"{P}/tab_construct_validity.tex", "w").write(body + "\\bottomrule\n\\end{tabular}\n\\par\\vspace{2pt}{\\footnotesize\\emph{Notes:} " + note + "}"); print("table tab_construct_validity.tex")70