TeX 74.5%
Python 25.4%
1"""206_extended.py -- Extended / rigorous tests for the AI-disclosure return study.3Produces results/tables/results2.json and several CSV/parquet artifacts.45Blocks:6 A. Placebo: replicate the post-2022 break on the 5 NON-AI semantic factors.7 B. Sector-neutral sorts and AIxSector / AIxSize double sorts.8 C. Magnificent-7 and mega-cap exclusion robustness.9 D. Alternative signal definitions (mean, top5pct) and horizons (6m, 24m).10 E. Year-by-year Fama-MacBeth slopes (for time-variation figure).11 F. PCA of the three AI signals -> AI_PC1; re-test.12 G. Difference-in-differences around ChatGPT (binary & continuous treatment).13 H. Forward realized volatility as an alternative performance outcome.14"""15import json, warnings16import numpy as np, pandas as pd17import statsmodels.api as sm, statsmodels.formula.api as smf18from statsmodels.stats.sandwich_covariance import cov_hac19warnings.filterwarnings("ignore")2021D="data/processed"; T="results/tables"22df=pd.read_parquet(f"{D}/analysis_panel.parquet")23mret=pd.read_parquet(f"{D}/monthly_returns.parquet")24mret["date"]=pd.to_datetime(mret["date"]).dt.to_period("M").dt.to_timestamp("M")25R2={}2627def nw_t(x,lags=3):28 x=pd.Series(x).dropna().values29 if len(x)==0: return np.nan,np.nan30 if len(x)<3: return float(x.mean()),np.nan31 r=sm.OLS(x,np.ones((len(x),1))).fit()32 se=np.sqrt(cov_hac(r,nlags=lags)[0,0]); return x.mean(),x.mean()/se3334def zс(g): return (g-g.mean())/g.std(ddof=0)3536def subperiod_spread(d, zcol, yrs, oc="fwd_ret"):37 sp=[]38 for yr,g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"):39 g=g.dropna(subset=[zcol,oc])40 if len(g)<25: continue41 g=g.copy(); g["q"]=pd.qcut(g[zcol],5,labels=False,duplicates="drop")+142 sp.append(g[g.q==5][oc].mean()-g[g.q==1][oc].mean())43 mu,t=nw_t(sp); return round(mu,4),round(t,2),len(sp)4445# =========================================================46# A. PLACEBO -- all 8 semantic factors, pre vs post + interaction47# =========================================================48FACTORS={"AI adoption":"ai_adoption_wmean","AI infrastructure":"ai_infrastructure_wmean",49 "AI disruption risk":"ai_disruption_risk_wmean","ESG commitment":"esg_commitment_wmean",50 "Climate risk":"climate_risk_wmean","Cybersecurity":"cybersecurity_wmean",51 "Automation/labor":"automation_labor_wmean","Macro risk":"macro_risk_wmean"}52placebo={}53for name,col in FACTORS.items():54 z=f"_z_{col}"55 df[z]=df.groupby("fiscal_year")[col].transform(zс)56 pre=subperiod_spread(df,z,range(2010,2022)); post=subperiod_spread(df,z,range(2022,2025))57 d2=df.dropna(subset=["fwd_w",z,"z_size","z_mom"]).copy()58 r=smf.ols(f"fwd_w ~ {z} * post + z_size + z_mom",data=d2).fit(cov_type="cluster",cov_kwds={"groups":d2["fiscal_year"]})59 iv=f"{z}:post"60 placebo[name]={"pre":pre,"post":post,"interaction":round(r.params[iv],4),"interaction_t":round(r.tvalues[iv],2),61 "is_ai":name.startswith("AI")}62R2["placebo"]=placebo63print("=== A. PLACEBO (Q5-Q1 spread pre vs post, interaction t) ===")64for n,v in placebo.items():65 print(f" {n:20s} pre={v['pre'][0]*100:+5.1f}pp post={v['post'][0]*100:+6.1f}pp(t={v['post'][1]:.2f}) interact_t={v['interaction_t']:+.2f}")6667# =========================================================68# B. SECTOR-NEUTRAL sorts + double sorts69# =========================================================70# sector-neutral: demean AI composite within sector-year, then sort71df["z_AI_sectneu"]=df.groupby(["fiscal_year","sector"])["ai_infrastructure_wmean"].transform(72 lambda g: zс(g) if g.std(ddof=0)>0 else g*0)73# use composite of adoption+infra sector-neutral74tmp_a=df.groupby(["fiscal_year","sector"])["ai_adoption_wmean"].transform(lambda g: zс(g) if g.std(ddof=0)>0 else g*0)75tmp_i=df.groupby(["fiscal_year","sector"])["ai_infrastructure_wmean"].transform(lambda g: zс(g) if g.std(ddof=0)>0 else g*0)76df["z_AI_COMPOS_sn"]=(tmp_a+tmp_i)/277sn={}78sn["full"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2010,2025))79sn["pre"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2010,2022))80sn["post"]=subperiod_spread(df,"z_AI_COMPOS_sn",range(2022,2025))81R2["sector_neutral"]=sn82print("\n=== B. SECTOR-NEUTRAL composite Q5-Q1 ===")83print(f" full={sn['full'][0]*100:+.1f}pp pre={sn['pre'][0]*100:+.1f}pp post={sn['post'][0]*100:+.1f}pp(t={sn['post'][1]})")8485# AI x Sector heatmap data: mean fwd_ret of high-AI minus low-AI within each sector, post-period86heat=[]87for (sec),g in df[df.fiscal_year>=2022].groupby("sector"):88 g=g.dropna(subset=["z_AI_COMPOS","fwd_ret"])89 if len(g)<15: continue90 med=g["z_AI_COMPOS"].median()91 hi=g[g.z_AI_COMPOS>med].fwd_ret.mean(); lo=g[g.z_AI_COMPOS<=med].fwd_ret.mean()92 heat.append({"sector":sec,"hi_minus_lo":(hi-lo)*100,"n":len(g)})93pd.DataFrame(heat).to_csv(f"{T}/t6_sector_aieffect_post.csv",index=False)9495# AI x Size double sort (post-period): 2x396def double_sort(d,zcol,bycol,oc="fwd_ret"):97 d=d.dropna(subset=[zcol,bycol,oc]).copy()98 d["sz"]=d.groupby("fiscal_year")[bycol].transform(lambda g: pd.qcut(g,2,labels=["Small","Large"],duplicates="drop"))99 d["ai"]=d.groupby("fiscal_year")[zcol].transform(lambda g: pd.qcut(g,3,labels=["Lo","Mid","Hi"],duplicates="drop"))100 return d.groupby(["sz","ai"],observed=True)[oc].mean().mul(100).round(2).unstack()101ds_post=double_sort(df[df.fiscal_year>=2022],"z_AI_COMPOS","log_mktcap")102ds_pre=double_sort(df[df.fiscal_year<2022],"z_AI_COMPOS","log_mktcap")103ds_post.to_csv(f"{T}/t6_double_sort_post.csv"); ds_pre.to_csv(f"{T}/t6_double_sort_pre.csv")104R2["double_sort_post"]=ds_post.to_dict(); R2["double_sort_pre"]=ds_pre.to_dict()105print("\n=== B. AIxSize double sort POST (mean fwd ret %) ===\n",ds_post.to_string())106107# =========================================================108# C. MAGNIFICENT-7 / mega-cap exclusion109# =========================================================110MAG7=["AAPL","MSFT","NVDA","AMZN","GOOGL","META","TSLA","GOOG","AVGO"]111def spread_excl(d,zcol,yrs,exclude=None,topn_drop=0):112 sp=[]113 for yr,g in d[d.fiscal_year.isin(list(yrs))].groupby("fiscal_year"):114 g=g.dropna(subset=[zcol,"fwd_ret"]).copy()115 if exclude: g=g[~g.ticker.isin(exclude)]116 if topn_drop: g=g[g.mktcap < g.mktcap.nlargest(topn_drop).min()]117 if len(g)<25: continue118 g["q"]=pd.qcut(g[zcol],5,labels=False,duplicates="drop")+1119 sp.append(g[g.q==5].fwd_ret.mean()-g[g.q==1].fwd_ret.mean())120 mu,t=nw_t(sp); return round(mu,4),round(t,2),len(sp)121excl={"baseline_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025)),122 "ex_mag7_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025),exclude=MAG7),123 "ex_top10_post":spread_excl(df,"z_AI_COMPOS",range(2022,2025),topn_drop=10),124 "ex_mag7_pre":spread_excl(df,"z_AI_COMPOS",range(2010,2022),exclude=MAG7)}125R2["mag7"]=excl126print("\n=== C. Mega-cap exclusion (composite Q5-Q1) ===")127for k,v in excl.items(): print(f" {k:18s} {v[0]*100:+.1f}pp (t={v[1]})")128129# =========================================================130# D. ALTERNATIVE signals & horizons131# =========================================================132# alt signal defs (mean, top5pct) for adoption+infra composite133for suff in ["mean","top5pct"]:134 a=df.groupby("fiscal_year")[f"ai_adoption_{suff}"].transform(zс)135 i=df.groupby("fiscal_year")[f"ai_infrastructure_{suff}"].transform(zс)136 df[f"z_AI_{suff}"]=(a+i)/2137alt_sig={}138for suff in ["wmean(base)","mean","top5pct"]:139 col="z_AI_COMPOS" if suff=="wmean(base)" else f"z_AI_{suff}"140 alt_sig[suff]={"pre":subperiod_spread(df,col,range(2010,2022)),"post":subperiod_spread(df,col,range(2022,2025))}141R2["alt_signal"]=alt_sig142print("\n=== D. Alt signal defs (composite Q5-Q1 pre|post) ===")143for k,v in alt_sig.items(): print(f" {k:14s} pre={v['pre'][0]*100:+.1f} post={v['post'][0]*100:+.1f}(t={v['post'][1]})")144145# alt horizons: compute 6m and 24m forward returns146px_by={t:g.sort_values("date") for t,g in mret.groupby("ticker")}147def cum(d,start,end,minn):148 m=d[(d.date>start)&(d.date<=end)]149 if m["ret"].notna().sum()<minn: return np.nan150 return (1+m["ret"].fillna(0)).prod()-1151for H,mm,minn in [(6,6,3),(24,24,12)]:152 vals=[]153 for _,r in df.iterrows():154 d=px_by.get(r.ticker)155 if d is None: vals.append(np.nan); continue156 form=pd.Timestamp(int(r.fiscal_year)+1,6,30)+pd.offsets.MonthEnd(0)157 end=form+pd.offsets.MonthEnd(mm)158 vals.append(cum(d,form,end,minn))159 df[f"fwd_ret_{H}m"]=vals160alt_hor={}161for H in [6,24]:162 alt_hor[f"{H}m"]={"pre":subperiod_spread(df,"z_AI_COMPOS",range(2010,2022),oc=f"fwd_ret_{H}m"),163 "post":subperiod_spread(df,"z_AI_COMPOS",range(2022,2024),oc=f"fwd_ret_{H}m")}164R2["alt_horizon"]=alt_hor165print("\n=== D. Alt horizons (composite Q5-Q1 pre|post) ===")166for k,v in alt_hor.items(): print(f" {k:4s} pre={v['pre'][0]*100:+.1f} post={v['post'][0]*100:+.1f}(t={v['post'][1]})")167168# =========================================================169# E. YEAR-BY-YEAR Fama-MacBeth slope (AI composite, controls)170# =========================================================171fm_year=[]172for yr,g in df.groupby("fiscal_year"):173 g=g.dropna(subset=["fwd_ret","z_AI_COMPOS","z_size","z_mom","z_vol"])174 if len(g)<30: continue175 X=sm.add_constant(g[["z_AI_COMPOS","z_size","z_mom","z_vol"]])176 r=sm.OLS(g["fwd_ret"],X).fit(cov_type="HC1")177 fm_year.append({"year":int(yr),"coef":r.params["z_AI_COMPOS"],"se":r.bse["z_AI_COMPOS"],"t":r.tvalues["z_AI_COMPOS"]})178pd.DataFrame(fm_year).to_csv(f"{T}/t6_fm_byyear.csv",index=False)179R2["fm_byyear"]=fm_year180print("\n=== E. Year-by-year FM slope on AI composite ===")181print(" "+", ".join(f"{x['year']}:{x['coef']*100:+.1f}" for x in fm_year))182183# =========================================================184# F. PCA of three AI signals -> AI_PC1185# =========================================================186from numpy.linalg import svd187Z=df[["z_AI_ADOPT","z_AI_INFRA","z_AI_RISK"]].dropna()188Zc=Z-Z.mean()189U,S,Vt=svd(Zc.values,full_matrices=False)190evr=(S**2)/np.sum(S**2)191pc1=Zc.values@Vt[0]192df.loc[Z.index,"z_AI_PC1"]=(pc1-pc1.mean())/pc1.std()193# orient so positive = more AI194if df[["z_AI_PC1","z_AI_ADOPT"]].corr().iloc[0,1]<0: df["z_AI_PC1"]*=-1195df["z_AI_PC1"]=df.groupby("fiscal_year")["z_AI_PC1"].transform(zс)196R2["pca"]={"explained_var_ratio":[round(float(x),3) for x in evr],197 "loadings_pc1":[round(float(x),3) for x in Vt[0]],198 "pre":subperiod_spread(df,"z_AI_PC1",range(2010,2022)),199 "post":subperiod_spread(df,"z_AI_PC1",range(2022,2025))}200print(f"\n=== F. PCA: EVR={R2['pca']['explained_var_ratio']} PC1 loadings={R2['pca']['loadings_pc1']}")201print(f" PC1 Q5-Q1 pre={R2['pca']['pre'][0]*100:+.1f} post={R2['pca']['post'][0]*100:+.1f}(t={R2['pca']['post'][1]})")202203# =========================================================204# G. DIFFERENCE-IN-DIFFERENCES around ChatGPT205# =========================================================206from linearmodels.panel import PanelOLS207# pre-period exposure fixed at firm's mean AI composite over 2019-2021208pre_exp=df[df.fiscal_year.isin([2019,2020,2021])].groupby("ticker")["ai_infrastructure_wmean"].mean()209exp_comp=df[df.fiscal_year.isin([2019,2020,2021])].groupby("ticker")[["ai_adoption_wmean","ai_infrastructure_wmean"]].mean().mean(axis=1)210df["pre_AI_cont"]=df.ticker.map((exp_comp-exp_comp.mean())/exp_comp.std())211df["treated"]=df.ticker.map((exp_comp>exp_comp.median()).astype(float))212did=df.dropna(subset=["fwd_w","pre_AI_cont","z_size","z_mom"]).copy()213did=did.set_index(["ticker","fiscal_year"])214# binary DiD: treated x post with entity+time FE (treated & post absorbed; interaction identified)215did["tp"]=did["treated"]*did["post"]216m_did=PanelOLS.from_formula("fwd_w ~ tp + z_size + z_mom + EntityEffects + TimeEffects",217 data=did,drop_absorbed=True).fit(cov_type="clustered",cluster_entity=True,cluster_time=True)218# continuous DiD219did["cp"]=did["pre_AI_cont"]*did["post"]220m_didc=PanelOLS.from_formula("fwd_w ~ cp + z_size + z_mom + EntityEffects + TimeEffects",221 data=did,drop_absorbed=True).fit(cov_type="clustered",cluster_entity=True,cluster_time=True)222R2["did"]={"binary":{"coef":round(m_did.params["tp"],4),"t":round(m_did.tstats["tp"],2),"n":int(m_did.nobs)},223 "continuous":{"coef":round(m_didc.params["cp"],4),"t":round(m_didc.tstats["cp"],2),"n":int(m_didc.nobs)}}224print("\n=== G. DiD around ChatGPT (firm+year FE, double-clustered) ===")225print(f" binary treated x post = {R2['did']['binary']['coef']:+.4f} (t={R2['did']['binary']['t']})")226print(f" continuous exposure x post = {R2['did']['continuous']['coef']:+.4f} (t={R2['did']['continuous']['t']})")227228# event-time DiD: continuous exposure interacted with each year dummy (ref=2021)229df["pre_AI_cont2"]=df["pre_AI_cont"]230ev=df.dropna(subset=["fwd_w","pre_AI_cont2","z_size","z_mom"]).copy()231ev=ev[ev.fiscal_year.between(2015,2024)]232ev["yr"]=ev.fiscal_year.astype(int).astype(str)233m_ev=smf.ols("fwd_w ~ C(yr, Treatment('2021')):pre_AI_cont2 + C(yr) + z_size + z_mom",data=ev).fit(234 cov_type="cluster",cov_kwds={"groups":ev["fiscal_year"]})235evt=[]236for yr in range(2015,2025):237 cand=[f"C(yr, Treatment('2021'))[{yr}]:pre_AI_cont2",238 f"C(yr, Treatment('2021'))[T.{yr}]:pre_AI_cont2"]239 key=next((k for k in cand if k in m_ev.params.index),None)240 if key:241 evt.append({"year":yr,"coef":m_ev.params[key],"se":m_ev.bse[key]})242 elif yr==2021:243 evt.append({"year":yr,"coef":0.0,"se":0.0})244pd.DataFrame(evt).to_csv(f"{T}/t6_did_eventtime.csv",index=False)245R2["did_eventtime"]=evt246print(" event-time slopes:", ", ".join(f"{e['year']}:{e['coef']*100:+.1f}" for e in evt))247248# =========================================================249# H. FORWARD VOLATILITY outcome250# =========================================================251fvol=[]252for _,r in df.reset_index(drop=True).iterrows():253 d=px_by.get(r.ticker)254 if d is None: fvol.append(np.nan); continue255 form=pd.Timestamp(int(r.fiscal_year)+1,6,30)+pd.offsets.MonthEnd(0)256 end=form+pd.offsets.MonthEnd(12)257 w=d[(d.date>form)&(d.date<=end)]258 fvol.append(w["ret"].std()*np.sqrt(12) if w["ret"].notna().sum()>=6 else np.nan)259df["fwd_vol"]=fvol260volr={}261for yrs,lab in [(range(2010,2022),"pre"),(range(2022,2025),"post")]:262 sp=[]263 for yr,g in df[df.fiscal_year.isin(list(yrs))].groupby("fiscal_year"):264 g=g.dropna(subset=["z_AI_COMPOS","fwd_vol"])265 if len(g)<25: continue266 g=g.copy(); g["q"]=pd.qcut(g.z_AI_COMPOS,5,labels=False,duplicates="drop")+1267 sp.append(g[g.q==5].fwd_vol.mean()-g[g.q==1].fwd_vol.mean())268 mu,t=nw_t(sp); volr[lab]=(round(mu,4),round(t,2),len(sp))269R2["fwd_vol"]=volr270print(f"\n=== H. Forward vol Q5-Q1: pre={volr['pre'][0]:+.3f} post={volr['post'][0]:+.3f}(t={volr['post'][1]})")271272json.dump(R2,open(f"{T}/results2.json","w"),indent=2,default=str)273df.to_parquet(f"{D}/analysis_panel.parquet")274print("\nSaved results2.json and updated analysis_panel.parquet")275