TeX 74.5%
Python 25.4%
1"""203_analysis.py -- Empirical asset-pricing analysis3Does the SEMANTIC INTENSITY of AI disclosure in 10-K filings explain the4cross-section of subsequent S&P 500 stock returns?56Signals (from fiscal year Y, section-weighted cosine similarity to AI anchors):7 AI_ADOPT = ai_adoption_wmean (internal AI use)8 AI_INFRA = ai_infrastructure_wmean (AI compute / capex exposure)9 AI_RISK = ai_disruption_risk_wmean(AI product-disruption risk)10 AI_COMPOS = z(AI_ADOPT)+z(AI_INFRA) (standardized opportunity composite)1112Outcome: forward 12-month return, July(Y+1)->June(Y+2) [strictly tradeable].1314Methods: (1) cross-sectional z-scoring within year; (2) quintile portfolio15sorts (EW & VW, raw & market-adjusted) with Newey-West t-stats; (3) long-short16spread regressed on FF5+Momentum to test for alpha; (4) two-way fixed-effects17panel regressions with double-clustered SE; (5) Fama-MacBeth; (6) a18post-ChatGPT (Nov-2022) structural-break interaction.19"""20import json, warnings21import numpy as np22import pandas as pd23import statsmodels.api as sm24import statsmodels.formula.api as smf25from statsmodels.stats.sandwich_covariance import cov_hac2627warnings.filterwarnings("ignore")28D = "data/processed"; T = "results/tables"29m = pd.read_parquet(f"{D}/merged_panel.parquet")30ret = pd.read_parquet(f"{D}/monthly_returns.parquet")31ff = pd.read_parquet(f"{D}/ff5_monthly.parquet")32mom = pd.read_parquet(f"{D}/mom_monthly.parquet")33RES = {}3435SIGNALS = {"AI_ADOPT": "ai_adoption_wmean",36 "AI_INFRA": "ai_infrastructure_wmean",37 "AI_RISK": "ai_disruption_risk_wmean"}3839df = m[m.fwd_ret.notna()].copy()4041# winsorize returns & controls at 1/99 within sample for regressions42def wins(s, lo=.01, hi=.99):43 ql, qh = s.quantile(lo), s.quantile(hi)44 return s.clip(ql, qh)45df["fwd_w"] = wins(df["fwd_ret"])46df["fwd_madj_w"] = wins(df["fwd_ret_mktadj"])47df["mom_w"] = wins(df["mom_12m"])48df["vol_w"] = wins(df["vol_12m"])4950# cross-sectional z-scores within formation year51def zscore(g):52 return (g - g.mean()) / g.std(ddof=0)53for k, col in SIGNALS.items():54 df[f"z_{k}"] = df.groupby("fiscal_year")[col].transform(zscore)55df["z_size"] = df.groupby("fiscal_year")["log_mktcap"].transform(zscore)56df["z_mom"] = df.groupby("fiscal_year")["mom_w"].transform(zscore)57df["z_vol"] = df.groupby("fiscal_year")["vol_w"].transform(zscore)58df["z_AI_COMPOS"] = df[["z_AI_ADOPT", "z_AI_INFRA"]].mean(axis=1)5960# ============================================================61# 1. SUMMARY STATISTICS62# ============================================================63summ_cols = list(SIGNALS.values()) + ["fwd_ret", "fwd_ret_mktadj", "mom_12m", "vol_12m", "log_mktcap"]64summary = df[summ_cols].describe(percentiles=[.05,.25,.5,.75,.95]).T65summary.to_csv(f"{T}/t1_summary_stats.csv")66RES["n_obs"] = int(len(df))67RES["n_firms"] = int(df.ticker.nunique())68RES["yrs"] = f"{int(df.fiscal_year.min())}-{int(df.fiscal_year.max())}"69print("N obs", len(df), "| firms", df.ticker.nunique(), "| formation yrs", RES["yrs"])7071# correlation among signals & with outcome72corr = df[["z_AI_ADOPT","z_AI_INFRA","z_AI_RISK","z_size","z_mom","fwd_ret"]].corr()73corr.to_csv(f"{T}/t1b_correlations.csv")7475# ============================================================76# 2. QUINTILE PORTFOLIO SORTS77# ============================================================78def newey_mean_t(x, lags=3):79 x = x.dropna().values80 if len(x) < 3: return np.nan, np.nan81 X = np.ones((len(x),1))82 res = sm.OLS(x, X).fit()83 se = np.sqrt(cov_hac(res, nlags=lags)[0,0])84 return x.mean(), x.mean()/se8586def sort_table(signal_z, weight=None, outcome="fwd_ret"):87 rows = []88 ann_means = {q: [] for q in range(1,6)}89 ann_ls = []90 for yr, g in df.groupby("fiscal_year"):91 g = g.dropna(subset=[signal_z, outcome])92 if len(g) < 25: continue93 g = g.copy()94 g["q"] = pd.qcut(g[signal_z], 5, labels=False, duplicates="drop") + 195 for q in range(1,6):96 sub = g[g.q==q]97 if sub.empty: continue98 if weight is None:99 ann_means[q].append(sub[outcome].mean())100 else:101 w = sub[weight].clip(lower=0).fillna(0)102 ann_means[q].append(np.average(sub[outcome], weights=w) if w.sum()>0 else sub[outcome].mean())103 if ann_means[5] and ann_means[1]:104 ann_ls.append(ann_means[5][-1] - ann_means[1][-1])105 out = {}106 for q in range(1,6):107 mu, t = newey_mean_t(pd.Series(ann_means[q]))108 out[f"Q{q}"] = mu109 mu_ls, t_ls = newey_mean_t(pd.Series(ann_ls))110 out["Q5_minus_Q1"] = mu_ls111 out["t_stat"] = t_ls112 out["n_years"] = len(ann_ls)113 return out114115sorts = {}116for k in ["AI_ADOPT","AI_INFRA","AI_RISK","AI_COMPOS"]:117 sorts[f"{k}_EW"] = sort_table(f"z_{k}", None, "fwd_ret")118 sorts[f"{k}_VW"] = sort_table(f"z_{k}", "mktcap", "fwd_ret")119 sorts[f"{k}_EW_madj"] = sort_table(f"z_{k}", None, "fwd_ret_mktadj")120sort_df = pd.DataFrame(sorts).T121sort_df.to_csv(f"{T}/t2_portfolio_sorts.csv")122RES["sorts"] = sort_df.round(4).to_dict("index")123print("\n=== Quintile sorts (forward 12m return) ===")124print(sort_df.round(4).to_string())125126# ============================================================127# 3. LONG-SHORT MONTHLY TIME SERIES + FACTOR ALPHA128# ============================================================129ff = ff.merge(mom[["date","MOM"]], on="date", how="left")130ff["date"] = pd.to_datetime(ff["date"]).dt.to_period("M").dt.to_timestamp("M")131ff_ix = ff.set_index("date")132ret = ret.copy(); ret["date"] = pd.to_datetime(ret["date"]).dt.to_period("M").dt.to_timestamp("M")133ret_by = {t: d.set_index("date")["ret"] for t, d in ret.groupby("ticker")}134135def ls_monthly(signal_z, n_groups=5):136 """Annually-rebalanced EW quintile portfolios, monthly returns."""137 monthly = {} # date -> dict(q->list of rets)138 for yr, g in df.groupby("fiscal_year"):139 g = g.dropna(subset=[signal_z])140 if len(g) < 25: continue141 g = g.copy(); g["q"] = pd.qcut(g[signal_z], n_groups, labels=False, duplicates="drop")+1142 hold = pd.date_range(pd.Timestamp(yr+1,7,31), pd.Timestamp(yr+2,6,30), freq="ME")143 for q in [1, n_groups]:144 tks = g[g.q==q].ticker.tolist()145 for dt in hold:146 r = [ret_by[t].get(dt, np.nan) for t in tks if t in ret_by]147 r = [x for x in r if pd.notna(x)]148 if r:149 monthly.setdefault(dt, {}).setdefault(q, []).append(np.mean(r))150 recs = []151 for dt, qd in sorted(monthly.items()):152 if 1 in qd and n_groups in qd:153 recs.append({"date": dt, "lo": np.mean(qd[1]), "hi": np.mean(qd[n_groups])})154 ts = pd.DataFrame(recs)155 ts["ls"] = ts["hi"] - ts["lo"]156 return ts157158def factor_alpha(ts, label):159 s = ts.merge(ff_ix.reset_index(), on="date", how="inner").dropna(subset=["ls","MktRF","SMB","HML","RMW","CMA","MOM"])160 y = s["ls"] - 0 # long-short is already self-financing (RF nets out)161 out = {}162 # CAPM163 X1 = sm.add_constant(s[["MktRF"]]); r1 = sm.OLS(y, X1).fit(cov_type="HAC", cov_kwds={"maxlags":3})164 out["capm_alpha_m"] = r1.params["const"]; out["capm_alpha_t"] = r1.tvalues["const"]165 # FF5+MOM166 X2 = sm.add_constant(s[["MktRF","SMB","HML","RMW","CMA","MOM"]]); r2 = sm.OLS(y, X2).fit(cov_type="HAC", cov_kwds={"maxlags":3})167 out["ff6_alpha_m"] = r2.params["const"]; out["ff6_alpha_t"] = r2.tvalues["const"]168 out["ff6_alpha_ann"] = (1+r2.params["const"])**12 - 1169 out["mean_ls_m"] = y.mean(); out["mean_ls_ann"] = (1+y.mean())**12-1170 out["n_months"] = len(s)171 out["betas"] = {k: round(r2.params[k],3) for k in ["MktRF","SMB","HML","RMW","CMA","MOM"]}172 RES.setdefault("ls_alpha", {})[label] = {k:(round(v,5) if isinstance(v,float) else v) for k,v in out.items()}173 print(f"\n[{label}] mean LS {out['mean_ls_ann']*100:.2f}%/yr | CAPM a={out['capm_alpha_m']*1200:.2f}%/yr (t={out['capm_alpha_t']:.2f}) "174 f"| FF6 a={out['ff6_alpha_ann']*100:.2f}%/yr (t={out['ff6_alpha_t']:.2f}) | n={out['n_months']}m")175 return ts, out176177ls_results = {}178for k in ["AI_ADOPT","AI_INFRA","AI_COMPOS","AI_RISK"]:179 ts = ls_monthly(f"z_{k}")180 ts.to_parquet(f"{D}/ls_ts_{k}.parquet", index=False)181 factor_alpha(ts, k)182183# ============================================================184# 4. TWO-WAY FIXED-EFFECTS PANEL REGRESSIONS (double-clustered SE)185# ============================================================186from linearmodels.panel import PanelOLS187pdf = df.dropna(subset=["fwd_w","z_AI_ADOPT","z_AI_INFRA","z_AI_RISK","z_size","z_mom","z_vol"]).copy()188pdf = pdf.set_index(["ticker","fiscal_year"])189190def fe_reg(formula, label):191 res = PanelOLS.from_formula(formula + " + EntityEffects + TimeEffects", data=pdf,192 drop_absorbed=True).fit(cov_type="clustered", cluster_entity=True, cluster_time=True)193 return res194195panel_specs = {196 "M1_adopt": "fwd_w ~ z_AI_ADOPT",197 "M2_adopt_ctrl": "fwd_w ~ z_AI_ADOPT + z_size + z_mom + z_vol",198 "M3_infra_ctrl": "fwd_w ~ z_AI_INFRA + z_size + z_mom + z_vol",199 "M4_risk_ctrl": "fwd_w ~ z_AI_RISK + z_size + z_mom + z_vol",200 "M5_all": "fwd_w ~ z_AI_ADOPT + z_AI_INFRA + z_AI_RISK + z_size + z_mom + z_vol",201}202print("\n=== Two-way FE panel (DV=forward 12m return, winsorized) ===")203panel_out = {}204for lab, f in panel_specs.items():205 r = fe_reg(f, lab)206 coefs = {}207 for v in r.params.index:208 coefs[v] = {"coef": round(r.params[v],4), "t": round(r.tstats[v],2), "p": round(r.pvalues[v],4)}209 panel_out[lab] = {"coefs": coefs, "n": int(r.nobs), "r2_within": round(r.rsquared_within,4)}210 key = [v for v in r.params.index if v.startswith("z_AI")]211 print(f"{lab}: " + ", ".join(f"{v}={r.params[v]:+.4f}(t={r.tstats[v]:.2f})" for v in key) + f" [n={int(r.nobs)}]")212RES["panel_fe"] = panel_out213214# ============================================================215# 5. FAMA-MACBETH216# ============================================================217def fama_macbeth(xvars, label):218 coefs = {v: [] for v in ["const"]+xvars}219 for yr, g in df.groupby("fiscal_year"):220 g = g.dropna(subset=["fwd_ret"]+xvars)221 if len(g) < 30: continue222 X = sm.add_constant(g[xvars]); y = g["fwd_ret"]223 r = sm.OLS(y, X).fit()224 for v in ["const"]+xvars:225 coefs[v].append(r.params.get(v, np.nan))226 out = {}227 for v in ["const"]+xvars:228 mu, t = newey_mean_t(pd.Series(coefs[v]))229 out[v] = {"mean_coef": round(mu,4), "t": round(t,2)}230 out["n_years"] = len(coefs["const"])231 RES.setdefault("fama_macbeth", {})[label] = out232 print(f"\n[FM {label}] " + ", ".join(f"{v}={out[v]['mean_coef']:+.4f}(t={out[v]['t']:.2f})" for v in xvars))233 return out234235print("\n=== Fama-MacBeth (DV=forward 12m return) ===")236fama_macbeth(["z_AI_ADOPT","z_size","z_mom","z_vol"], "adopt")237fama_macbeth(["z_AI_INFRA","z_size","z_mom","z_vol"], "infra")238fama_macbeth(["z_AI_ADOPT","z_AI_INFRA","z_AI_RISK","z_size","z_mom","z_vol"], "all")239240# ============================================================241# 6. POST-CHATGPT STRUCTURAL BREAK (formation years >= 2022)242# ============================================================243df["post"] = (df.fiscal_year >= 2022).astype(int)244print("\n=== Post-ChatGPT interaction (pooled OLS, year-clustered) ===")245brk = {}246for k in ["AI_ADOPT","AI_INFRA","AI_COMPOS"]:247 d2 = df.dropna(subset=["fwd_w",f"z_{k}","z_size","z_mom"]).copy()248 r = smf.ols(f"fwd_w ~ z_{k} * post + z_size + z_mom", data=d2).fit(249 cov_type="cluster", cov_kwds={"groups": d2["fiscal_year"]})250 iv = f"z_{k}:post"251 brk[k] = {"pre_slope": round(r.params[f"z_{k}"],4),252 "interaction": round(r.params[iv],4),253 "interaction_t": round(r.tvalues[iv],2),254 "post_slope": round(r.params[f"z_{k}"]+r.params[iv],4)}255 print(f"{k}: pre={brk[k]['pre_slope']:+.4f} | post-2022 extra={brk[k]['interaction']:+.4f} (t={brk[k]['interaction_t']:.2f}) | post total={brk[k]['post_slope']:+.4f}")256RES["structural_break"] = brk257258# pre/post quintile spread comparison for headline259def spread_subperiod(signal_z, yrs):260 sp=[]261 for yr,g in df[df.fiscal_year.isin(yrs)].groupby("fiscal_year"):262 g=g.dropna(subset=[signal_z,"fwd_ret"])263 if len(g)<25: continue264 g=g.copy(); g["q"]=pd.qcut(g[signal_z],5,labels=False,duplicates="drop")+1265 sp.append(g[g.q==5].fwd_ret.mean()-g[g.q==1].fwd_ret.mean())266 mu,t=newey_mean_t(pd.Series(sp)); return round(mu,4),round(t,2),len(sp)267RES["subperiod_spread"]={}268for k in ["AI_ADOPT","AI_INFRA","AI_COMPOS"]:269 pre=spread_subperiod(f"z_{k}", range(2010,2022))270 post=spread_subperiod(f"z_{k}", range(2022,2025))271 RES["subperiod_spread"][k]={"pre_2010_2021":pre,"post_2022_2024":post}272 print(f"{k} Q5-Q1: pre(10-21)={pre[0]:+.4f}(t={pre[1]}) | post(22-24)={post[0]:+.4f}(t={post[1]})")273274with open(f"{T}/results.json","w") as fh:275 json.dump(RES, fh, indent=2, default=str)276df.to_parquet(f"{D}/analysis_panel.parquet")277print("\nSaved results.json and analysis_panel.parquet")278