""" 06_extended.py -- Extended / rigorous tests for the AI-disclosure return study. Produces results/tables/results2.json and several CSV/parquet artifacts. Blocks: A. Placebo: replicate the post-2022 break on the 5 NON-AI semantic factors. B. Sector-neutral sorts and AIxSector / AIxSize double sorts. C. Magnificent-7 and mega-cap exclusion robustness. D. Alternative signal definitions (mean, top5pct) and horizons (6m, 24m). E. Year-by-year Fama-MacBeth slopes (for time-variation figure). F. PCA of the three AI signals -> AI_PC1; re-test. G. Difference-in-differences around ChatGPT (binary & continuous treatment). H. Forward realized volatility as an alternative performance outcome. """ import json, warnings import numpy as np, pandas as pd import statsmodels.api as sm, statsmodels.formula.api as smf from statsmodels.stats.sandwich_covariance import cov_hac warnings.filterwarnings("ignore") D="data/processed"; T="results/tables" df=pd.read_parquet(f"{D}/analysis_panel.parquet") mret=pd.read_parquet(f"{D}/monthly_returns.parquet") mret["date"]=pd.to_datetime(mret["date"]).dt.to_period("M").dt.to_timestamp("M") R2={} def nw_t(x,lags=3): x=pd.Series(x).dropna().values if len(x)==0: return np.nan,np.nan if len(x)<3: return float(x.mean()),np.nan r=sm.OLS(x,np.ones((len(x),1))).fit() se=np.sqrt(cov_hac(r,nlags=lags)[0,0]); return x.mean(),x.mean()/se def zс(g): return (g-g.mean())/g.std(ddof=0) def subperiod_spread(d, zcol, yrs, oc="fwd_ret"): sp=[] for yr,g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"): g=g.dropna(subset=[zcol,oc]) if len(g)<25: continue g=g.copy(); g["q"]=pd.qcut(g[zcol],5,labels=False,duplicates="drop")+1 sp.append(g[g.q==5][oc].mean()-g[g.q==1][oc].mean()) mu,t=nw_t(sp); return round(mu,4),round(t,2),len(sp) # ========================================================= # A. PLACEBO -- all 8 semantic factors, pre vs post + interaction # ========================================================= FACTORS={"AI adoption":"ai_adoption_wmean","AI infrastructure":"ai_infrastructure_wmean", "AI disruption risk":"ai_disruption_risk_wmean","ESG commitment":"esg_commitment_wmean", "Climate risk":"climate_risk_wmean","Cybersecurity":"cybersecurity_wmean", "Automation/labor":"automation_labor_wmean","Macro risk":"macro_risk_wmean"} placebo={} for name,col in FACTORS.items(): z=f"_z_{col}" df[z]=df.groupby("fiscal_year")[col].transform(zс) pre=subperiod_spread(df,z,range(2010,2022)); post=subperiod_spread(df,z,range(2022,2025)) d2=df.dropna(subset=["fwd_w",z,"z_size","z_mom"]).copy() r=smf.ols(f"fwd_w ~ {z} * post + z_size + z_mom",data=d2).fit(cov_type="cluster",cov_kwds={"groups":d2["fiscal_year"]}) iv=f"{z}:post" placebo[name]={"pre":pre,"post":post,"interaction":round(r.params[iv],4),"interaction_t":round(r.tvalues[iv],2), "is_ai":name.startswith("AI")} R2["placebo"]=placebo print("=== A. PLACEBO (Q5-Q1 spread pre vs post, interaction t) ===") for n,v in placebo.items(): print(f" {n:20s} pre={v['pre'][0]*100:+5.1f}pp post={v['post'][0]*100:+6.1f}pp(t={v['post'][1]:.2f}) interact_t={v['interaction_t']:+.2f}") # ========================================================= # B. SECTOR-NEUTRAL sorts + double sorts # ========================================================= # sector-neutral: demean AI composite within sector-year, then sort df["z_AI_sectneu"]=df.groupby(["fiscal_year","sector"])["ai_infrastructure_wmean"].transform( lambda g: zс(g) if g.std(ddof=0)>0 else g*0) # use composite of adoption+infra sector-neutral tmp_a=df.groupby(["fiscal_year","sector"])["ai_adoption_wmean"].transform(lambda g: zс(g) if g.std(ddof=0)>0 else g*0) tmp_i=df.groupby(["fiscal_year","sector"])["ai_infrastructure_wmean"].transform(lambda g: zс(g) if g.std(ddof=0)>0 else g*0) df["z_AI_COMPOS_sn"]=(tmp_a+tmp_i)/2 sn={} sn["full"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2010,2025)) sn["pre"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2010,2022)) sn["post"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2022,2025)) R2["sector_neutral"]=sn print("\n=== B. SECTOR-NEUTRAL composite Q5-Q1 ===") print(f" full={sn['full'][0]*100:+.1f}pp pre={sn['pre'][0]*100:+.1f}pp post={sn['post'][0]*100:+.1f}pp(t={sn['post'][1]})") # AI x Sector heatmap data: mean fwd_ret of high-AI minus low-AI within each sector, post-period heat=[] for (sec),g in df[df.fiscal_year>=2022].groupby("sector"): g=g.dropna(subset=["z_AI_COMPOS","fwd_ret"]) if len(g)<15: continue med=g["z_AI_COMPOS"].median() hi=g[g.z_AI_COMPOS>med].fwd_ret.mean(); lo=g[g.z_AI_COMPOS<=med].fwd_ret.mean() heat.append({"sector":sec,"hi_minus_lo":(hi-lo)*100,"n":len(g)}) pd.DataFrame(heat).to_csv(f"{T}/t6_sector_aieffect_post.csv",index=False) # AI x Size double sort (post-period): 2x3 def double_sort(d,zcol,bycol,oc="fwd_ret"): d=d.dropna(subset=[zcol,bycol,oc]).copy() d["sz"]=d.groupby("fiscal_year")[bycol].transform(lambda g: pd.qcut(g,2,labels=["Small","Large"],duplicates="drop")) d["ai"]=d.groupby("fiscal_year")[zcol].transform(lambda g: pd.qcut(g,3,labels=["Lo","Mid","Hi"],duplicates="drop")) return d.groupby(["sz","ai"],observed=True)[oc].mean().mul(100).round(2).unstack() ds_post=double_sort(df[df.fiscal_year>=2022],"z_AI_COMPOS","log_mktcap") ds_pre=double_sort(df[df.fiscal_year<2022],"z_AI_COMPOS","log_mktcap") ds_post.to_csv(f"{T}/t6_double_sort_post.csv"); ds_pre.to_csv(f"{T}/t6_double_sort_pre.csv") R2["double_sort_post"]=ds_post.to_dict(); R2["double_sort_pre"]=ds_pre.to_dict() print("\n=== B. AIxSize double sort POST (mean fwd ret %) ===\n",ds_post.to_string()) # ========================================================= # C. MAGNIFICENT-7 / mega-cap exclusion # ========================================================= MAG7=["AAPL","MSFT","NVDA","AMZN","GOOGL","META","TSLA","GOOG","AVGO"] def spread_excl(d,zcol,yrs,exclude=None,topn_drop=0): sp=[] for yr,g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"): g=g.dropna(subset=[zcol,"fwd_ret"]).copy() if exclude: g=g[~g.ticker.isin(exclude)] if topn_drop: g=g[g.mktcap < g.mktcap.nlargest(topn_drop).min()] if len(g)<25: continue g["q"]=pd.qcut(g[zcol],5,labels=False,duplicates="drop")+1 sp.append(g[g.q==5].fwd_ret.mean()-g[g.q==1].fwd_ret.mean()) mu,t=nw_t(sp); return round(mu,4),round(t,2),len(sp) excl={"baseline_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025)), "ex_mag7_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025),exclude=MAG7), "ex_top10_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025),topn_drop=10), "ex_mag7_pre":spread_excl(df,"z_AI_COMPOS",range(2010,2022),exclude=MAG7)} R2["mag7"]=excl print("\n=== C. Mega-cap exclusion (composite Q5-Q1) ===") for k,v in excl.items(): print(f" {k:18s} {v[0]*100:+.1f}pp (t={v[1]})") # ========================================================= # D. ALTERNATIVE signals & horizons # ========================================================= # alt signal defs (mean, top5pct) for adoption+infra composite for suff in ["mean","top5pct"]: a=df.groupby("fiscal_year")[f"ai_adoption_{suff}"].transform(zс) i=df.groupby("fiscal_year")[f"ai_infrastructure_{suff}"].transform(zс) df[f"z_AI_{suff}"]=(a+i)/2 alt_sig={} for suff in ["wmean(base)","mean","top5pct"]: col="z_AI_COMPOS" if suff=="wmean(base)" else f"z_AI_{suff}" alt_sig[suff]={"pre":subperiod_spread(df,col,range(2010,2022)),"post":subperiod_spread(df,col,range(2022,2025))} R2["alt_signal"]=alt_sig print("\n=== D. Alt signal defs (composite Q5-Q1 pre|post) ===") for k,v in alt_sig.items(): print(f" {k:14s} pre={v['pre'][0]*100:+.1f} post={v['post'][0]*100:+.1f}(t={v['post'][1]})") # alt horizons: compute 6m and 24m forward returns px_by={t:g.sort_values("date") for t,g in mret.groupby("ticker")} def cum(d,start,end,minn): m=d[(d.date>start)&(d.date<=end)] if m["ret"].notna().sum() AI_PC1 # ========================================================= from numpy.linalg import svd Z=df[["z_AI_ADOPT","z_AI_INFRA","z_AI_RISK"]].dropna() Zc=Z-Z.mean() U,S,Vt=svd(Zc.values,full_matrices=False) evr=(S**2)/np.sum(S**2) pc1=Zc.values@Vt[0] df.loc[Z.index,"z_AI_PC1"]=(pc1-pc1.mean())/pc1.std() # orient so positive = more AI if df[["z_AI_PC1","z_AI_ADOPT"]].corr().iloc[0,1]<0: df["z_AI_PC1"]*=-1 df["z_AI_PC1"]=df.groupby("fiscal_year")["z_AI_PC1"].transform(zс) R2["pca"]={"explained_var_ratio":[round(float(x),3) for x in evr], "loadings_pc1":[round(float(x),3) for x in Vt[0]], "pre":subperiod_spread(df,"z_AI_PC1",range(2010,2022)), "post":subperiod_spread(df,"z_AI_PC1",range(2022,2025))} print(f"\n=== F. PCA: EVR={R2['pca']['explained_var_ratio']} PC1 loadings={R2['pca']['loadings_pc1']}") print(f" PC1 Q5-Q1 pre={R2['pca']['pre'][0]*100:+.1f} post={R2['pca']['post'][0]*100:+.1f}(t={R2['pca']['post'][1]})") # ========================================================= # G. DIFFERENCE-IN-DIFFERENCES around ChatGPT # ========================================================= from linearmodels.panel import PanelOLS # pre-period exposure fixed at firm's mean AI composite over 2019-2021 pre_exp=df[df.fiscal_year.isin([2019,2020,2021])].groupby("ticker")["ai_infrastructure_wmean"].mean() exp_comp=df[df.fiscal_year.isin([2019,2020,2021])].groupby("ticker")[["ai_adoption_wmean","ai_infrastructure_wmean"]].mean().mean(axis=1) df["pre_AI_cont"]=df.ticker.map((exp_comp-exp_comp.mean())/exp_comp.std()) df["treated"]=df.ticker.map((exp_comp>exp_comp.median()).astype(float)) did=df.dropna(subset=["fwd_w","pre_AI_cont","z_size","z_mom"]).copy() did=did.set_index(["ticker","fiscal_year"]) # binary DiD: treated x post with entity+time FE (treated & post absorbed; interaction identified) did["tp"]=did["treated"]*did["post"] m_did=PanelOLS.from_formula("fwd_w ~ tp + z_size + z_mom + EntityEffects + TimeEffects", data=did,drop_absorbed=True).fit(cov_type="clustered",cluster_entity=True,cluster_time=True) # continuous DiD did["cp"]=did["pre_AI_cont"]*did["post"] m_didc=PanelOLS.from_formula("fwd_w ~ cp + z_size + z_mom + EntityEffects + TimeEffects", data=did,drop_absorbed=True).fit(cov_type="clustered",cluster_entity=True,cluster_time=True) R2["did"]={"binary":{"coef":round(m_did.params["tp"],4),"t":round(m_did.tstats["tp"],2),"n":int(m_did.nobs)}, "continuous":{"coef":round(m_didc.params["cp"],4),"t":round(m_didc.tstats["cp"],2),"n":int(m_didc.nobs)}} print("\n=== G. DiD around ChatGPT (firm+year FE, double-clustered) ===") print(f" binary treated x post = {R2['did']['binary']['coef']:+.4f} (t={R2['did']['binary']['t']})") print(f" continuous exposure x post = {R2['did']['continuous']['coef']:+.4f} (t={R2['did']['continuous']['t']})") # event-time DiD: continuous exposure interacted with each year dummy (ref=2021) df["pre_AI_cont2"]=df["pre_AI_cont"] ev=df.dropna(subset=["fwd_w","pre_AI_cont2","z_size","z_mom"]).copy() ev=ev[ev.fiscal_year.between(2015,2024)] ev["yr"]=ev.fiscal_year.astype(int).astype(str) m_ev=smf.ols("fwd_w ~ C(yr, Treatment('2021')):pre_AI_cont2 + C(yr) + z_size + z_mom",data=ev).fit( cov_type="cluster",cov_kwds={"groups":ev["fiscal_year"]}) evt=[] for yr in range(2015,2025): cand=[f"C(yr, Treatment('2021'))[{yr}]:pre_AI_cont2", f"C(yr, Treatment('2021'))[T.{yr}]:pre_AI_cont2"] key=next((k for k in cand if k in m_ev.params.index),None) if key: evt.append({"year":yr,"coef":m_ev.params[key],"se":m_ev.bse[key]}) elif yr==2021: evt.append({"year":yr,"coef":0.0,"se":0.0}) pd.DataFrame(evt).to_csv(f"{T}/t6_did_eventtime.csv",index=False) R2["did_eventtime"]=evt print(" event-time slopes:", ", ".join(f"{e['year']}:{e['coef']*100:+.1f}" for e in evt)) # ========================================================= # H. FORWARD VOLATILITY outcome # ========================================================= fvol=[] for _,r in df.reset_index(drop=True).iterrows(): d=px_by.get(r.ticker) if d is None: fvol.append(np.nan); continue form=pd.Timestamp(int(r.fiscal_year)+1,6,30)+pd.offsets.MonthEnd(0) end=form+pd.offsets.MonthEnd(12) w=d[(d.date>form)&(d.date<=end)] fvol.append(w["ret"].std()*np.sqrt(12) if w["ret"].notna().sum()>=6 else np.nan) df["fwd_vol"]=fvol volr={} for yrs,lab in [(range(2010,2022),"pre"),(range(2022,2025),"post")]: sp=[] for yr,g in df[df.fiscal_year.isin(list(yrs))].groupby("fiscal_year"): g=g.dropna(subset=["z_AI_COMPOS","fwd_vol"]) if len(g)<25: continue g=g.copy(); g["q"]=pd.qcut(g.z_AI_COMPOS,5,labels=False,duplicates="drop")+1 sp.append(g[g.q==5].fwd_vol.mean()-g[g.q==1].fwd_vol.mean()) mu,t=nw_t(sp); volr[lab]=(round(mu,4),round(t,2),len(sp)) R2["fwd_vol"]=volr print(f"\n=== H. Forward vol Q5-Q1: pre={volr['pre'][0]:+.3f} post={volr['post'][0]:+.3f}(t={volr['post'][1]})") json.dump(R2,open(f"{T}/results2.json","w"),indent=2,default=str) df.to_parquet(f"{D}/analysis_panel.parquet") print("\nSaved results2.json and updated analysis_panel.parquet")