"""r3k_23_construct_validity.py — Stage D (FIRC-0002, package N1 laptop part / L7, cluster J: EDIT-05, AI_C-07, DATA-11, THEO-14, ADVE-03, AI_C-17).
Construct-validity rows for Tables 5 / 7 from the node audit outputs (results/n1/, N1 executor on M4M64a): (i) the keyword-count AI
baseline (published AI lexicon regex over the section texts: kw_core_wmean = core-term hits per 1,000 words, section-weighted) and
(ii) the alternative-anchor composite under AR-FIRC0002-2 (anchors stripped of cybersecurity / automation terms, scored against the
STORED chunk embeddings). Each signal is standardized within year, sorted into quintiles (raw and winsorized forward returns) and
entered in a Fama-MacBeth regression; rank correlations with the baseline composite are reported. Also the Utilities / Energy
exclusion bound for Tables 5 / 9. Writes results/r3k_v2/results_construct.json and paper/tables/tab_construct_validity.tex.
Run: .venv-review/bin/python -I scripts/r3k_23_construct_validity.py
"""
import os, sys, json, numpy as np, pandas as pd, statsmodels.api as sm
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import r3k_inference as I
V = "data/processed_r3k_v2"; T = "results/r3k_v2"; P = "paper/tables"; PRE, POST, ALL = range(2010, 2022), range(2022, 2025), range(2010, 2025)
df = pd.read_parquet(f"{V}/analysis_panel_v2.parquet"); RES = {"script": "r3k_23_construct_validity.py", "inputs": []}
def zc(g): return (g - g.mean()) / g.std(ddof=0)
kw = pd.read_csv("results/n1/keyword_firm_year.csv", dtype={"cik": str}) if os.path.exists("results/n1/keyword_firm_year.csv") else None
alt = pd.read_csv("results/n1/alt_firm_year.csv", dtype={"cik": str}) if os.path.exists("results/n1/alt_firm_year.csv") else None
if kw is not None:
    kw["cik"] = kw.cik.str.zfill(10); df = df.merge(kw[["cik", "fiscal_year", "kw_core_wmean", "kw_core_mean", "any_core_share", "kw_core_10k"]], on=["cik", "fiscal_year"], how="left"); RES["inputs"].append("results/n1/keyword_firm_year.csv")
    df["z_KW"] = df.groupby("fiscal_year")["kw_core_wmean"].transform(zc); df["z_KWlog"] = df.groupby("fiscal_year")["kw_core_wmean"].transform(lambda s: zc(np.log1p(s)))
if alt is not None:
    alt["cik"] = alt.cik.str.zfill(10); df = df.merge(alt[["cik", "fiscal_year", "alt_ai_adoption_strip_wmean", "alt_ai_infrastructure_strip_wmean", "alt_ai_adoption_core_wmean", "alt_cybersecurity_strip_wmean", "alt_automation_labor_strip_wmean"]], on=["cik", "fiscal_year"], how="left"); RES["inputs"].append("results/n1/alt_firm_year.csv")
    df["z_ALT"] = (df.groupby("fiscal_year")["alt_ai_adoption_strip_wmean"].transform(zc) + df.groupby("fiscal_year")["alt_ai_infrastructure_strip_wmean"].transform(zc)) / 2; df["z_ALT"] = df.groupby("fiscal_year")["z_ALT"].transform(zc)
    df["z_ALTcore"] = df.groupby("fiscal_year")["alt_ai_adoption_core_wmean"].transform(zc); df["z_CYBERstrip"] = df.groupby("fiscal_year")["alt_cybersecurity_strip_wmean"].transform(zc)
def sort_table(d, z, oc, yrs=ALL, minn=40):
    ls = {}
    for yr, g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"):
        g = g.dropna(subset=[z, oc]).copy()
        if len(g) < minn: continue
        g["q"] = pd.qcut(g[z].rank(method="first"), 5, labels=False) + 1; ls[int(yr)] = float(g[g.q == 5][oc].mean() - g[g.q == 1][oc].mean())
    v = list(ls.values()); mu, t = I.nw_t(v); return {"mean": round(mu, 4) if v else None, "t": round(t, 2) if t == t else None, "t_plain": round(I.plain_t(v), 2) if len(v) > 1 else None, "ci95": [round(x, 4) for x in I.ci95(v)] if len(v) > 1 else None, "n": len(v), "by_cohort": {k: round(x, 4) for k, x in ls.items()}}
def fm(d, z, oc):
    c = []
    for yr, g in d.groupby("fiscal_year"):
        g = g.dropna(subset=[oc, z, "z_size", "z_mom", "z_vol"])
        if len(g) < 40: continue
        c.append(sm.OLS(g[oc], sm.add_constant(g[[z, "z_size", "z_mom", "z_vol"]])).fit().params[z])
    mu, t = I.nw_t(c); return {"slope_pct": round(mu * 100, 2), "t": round(t, 2), "t_plain": round(I.plain_t(c), 2), "ci95_pct": [round(x * 100, 2) for x in I.ci95(c)], "n_years": len(c)}
SIG = [("z_AI_COMPOS", "AI composite (baseline)")] + ([("z_KW", "Keyword baseline (core AI terms per 1,000 words)"), ("z_KWlog", "Keyword baseline, log(1 + x)")] if kw is not None else []) + ([("z_ALT", "Stripped-anchor composite (AR-2)"), ("z_ALTcore", "Core-AI adoption anchor (AR-2)"), ("z_CYBERstrip", "Cybersecurity anchor stripped of AI terms (AR-2)")] if alt is not None else [])
RES["signals"] = {}
for z, l in SIG:
    d = df.dropna(subset=[z]); RES["signals"][z] = {"label": l, "n": int(len(d)), "spearman_with_composite": round(float(d[[z, "z_AI_COMPOS"]].corr(method="spearman").iloc[0, 1]), 3), "share_zero": round(float((d[z.replace("z_KW", "kw_core_wmean")] == 0).mean()), 3) if z == "z_KW" else None,
                        "full": sort_table(d, z, "fwd_ret"), "full_w": sort_table(d, z, "fwd_w"), "pre_w": sort_table(d, z, "fwd_w", PRE), "post_w": sort_table(d, z, "fwd_w", POST), "fm": fm(d, z, "fwd_ret"), "fm_w": fm(d, z, "fwd_w")}
    print(l, "| rho", RES["signals"][z]["spearman_with_composite"], "| full w", RES["signals"][z]["full_w"]["mean"], RES["signals"][z]["full_w"]["t"], "| FM w", RES["signals"][z]["fm_w"], flush=True)
# horse race composite vs keyword / stripped anchors (FM, winsorized)
def fm_multi(d, zs, oc="fwd_w"):
    c = {z: [] for z in zs}
    for yr, g in d.groupby("fiscal_year"):
        g = g.dropna(subset=[oc] + zs + ["z_size", "z_mom", "z_vol"])
        if len(g) < 40: continue
        r = sm.OLS(g[oc], sm.add_constant(g[zs + ["z_size", "z_mom", "z_vol"]])).fit()
        for z in zs: c[z].append(r.params[z])
    return {z: {"slope_pct": round(I.nw_t(v)[0] * 100, 2), "t": round(I.nw_t(v)[1], 2), "t_plain": round(I.plain_t(v), 2)} for z, v in c.items()}
RES["horse_race_w"] = {}
if kw is not None: RES["horse_race_w"]["composite_vs_keyword"] = fm_multi(df, ["z_AI_COMPOS", "z_KW"])
if alt is not None: RES["horse_race_w"]["composite_vs_stripped"] = fm_multi(df, ["z_AI_COMPOS", "z_ALT"])
# Utilities / Energy exclusion bound
ex = df[~df.sector.isin(["Utilities", "Energy"])]; RES["ex_utilities_energy"] = {"n": int(len(ex)), "full": sort_table(ex, "z_AI_COMPOS", "fwd_ret"), "full_w": sort_table(ex, "z_AI_COMPOS", "fwd_w"), "pre_w": sort_table(ex, "z_AI_COMPOS", "fwd_w", PRE), "post_w": sort_table(ex, "z_AI_COMPOS", "fwd_w", POST), "fm_w": fm(ex, "z_AI_COMPOS", "fwd_w"), "fm": fm(ex, "z_AI_COMPOS", "fwd_ret")}
# top-of-ranking sector composition (IDA / HE question)
top = df[df.groupby("fiscal_year")["z_AI_COMPOS"].transform(lambda s: s >= s.quantile(.8))]; RES["q5_sector_shares"] = top.sector.value_counts(normalize=True).round(3).head(8).to_dict(); RES["utilities_share_q5_vs_all"] = [round(float((top.sector == "Utilities").mean()), 3), round(float((df.sector == "Utilities").mean()), 3)]
json.dump(RES, open(f"{T}/results_construct.json", "w"), indent=2, default=str)
def pp(x): return "--" if x is None else f"{x * 100:+.1f}"
def tv(t): return "--" if t is None or (isinstance(t, float) and np.isnan(t)) else f"{t:.2f}"
body = "\\begin{tabular}{lrrrrrrrr}\n\\toprule\nSignal & $N$ & $\\rho$ with composite & Full Q5$-$Q1 raw & Full Q5$-$Q1 wins. (NW $t$ / plain $t$) & Pre wins. & Post wins. & FM slope raw (NW $t$) & FM slope wins. (NW $t$) \\\\\n\\midrule\n"
for z, l in SIG:
    s = RES["signals"][z]; body += f"{l} & {s['n']:,} & {s['spearman_with_composite']:.2f} & {pp(s['full']['mean'])} & {pp(s['full_w']['mean'])} ({tv(s['full_w']['t'])} / {tv(s['full_w']['t_plain'])}) & {pp(s['pre_w']['mean'])} & {pp(s['post_w']['mean'])} & {s['fm']['slope_pct']:+.2f} ({tv(s['fm']['t'])}) & {s['fm_w']['slope_pct']:+.2f} ({tv(s['fm_w']['t'])}) \\\\\n".replace(",", "{,}", 1) if False else f"{l} & {s['n']} & {s['spearman_with_composite']:.2f} & {pp(s['full']['mean'])} & {pp(s['full_w']['mean'])} ({tv(s['full_w']['t'])} / {tv(s['full_w']['t_plain'])}) & {pp(s['pre_w']['mean'])} & {pp(s['post_w']['mean'])} & {s['fm']['slope_pct']:+.2f} ({tv(s['fm']['t'])}) & {s['fm_w']['slope_pct']:+.2f} ({tv(s['fm_w']['t'])}) \\\\\n"
e = RES["ex_utilities_energy"]; body += f"AI composite, excluding Utilities and Energy & {e['n']} & & {pp(e['full']['mean'])} & {pp(e['full_w']['mean'])} ({tv(e['full_w']['t'])} / {tv(e['full_w']['t_plain'])}) & {pp(e['pre_w']['mean'])} & {pp(e['post_w']['mean'])} & {e['fm']['slope_pct']:+.2f} ({tv(e['fm']['t'])}) & {e['fm_w']['slope_pct']:+.2f} ({tv(e['fm_w']['t'])}) \\\\\n\\midrule\n"
for k, v in RES["horse_race_w"].items(): body += f"Horse race (FM, wins.): {k.replace('_', ' ')} & \\multicolumn{{8}}{{l}}{{" + "; ".join(f"{z}: {x['slope_pct']:+.2f} ({x['t']:.2f})" for z, x in v.items()) + "} \\\\\n"
note = f"Keyword baseline from the node audit (code/n1, published AI lexicon over the section texts; share of firm-years with zero core hits {RES['signals'].get('z_KW', {}).get('share_zero', '--')}); stripped-anchor composite = anchors without the cybersecurity / automation vocabulary, scored against the stored chunk embeddings (AR-FIRC0002-2, no corpus re-embedding). Utilities are {RES['utilities_share_q5_vs_all'][0] * 100:.1f}\\% of top-quintile firm-years vs {RES['utilities_share_q5_vs_all'][1] * 100:.1f}\\% of the sample. NW $t$ over cohorts (3 lags); FM controls size, momentum, volatility."
open(f"{P}/tab_construct_validity.tex", "w").write(body + "\\bottomrule\n\\end{tabular}\n\\par\\vspace{2pt}{\\footnotesize\\emph{Notes:} " + note + "}"); print("table tab_construct_validity.tex")