1#!/usr/bin/env python32"""Stage E (Agent 10, writing) — supplementary number macros for the manuscript prose.34Every macro below is read from a result file produced by a Stage D / N1 script; nothing is typed.5Sources: results/n1/tokenizer_audit.json, keyword_baseline.json, keyword_chunk_deciles.csv,6keyword_corr_by_year.csv, alt_anchor_results.json; results/r3k_v2/results_construct.json,7results_surv.json, results2.json, mktcap_diag.json, validation/spid_sample_meta.json;8the committed baseline results/r3k/*.json (HEAD 3cf7560) for the BEFORE values.9Output: paper/sections/macros_stageE.tex (\\input from paper/main.tex after tables/macros.tex).10Usage: python3 -I scripts/stageE_writing_macros.py (run from the repository root)11"""12from __future__ import annotations1314import csv15import json16import math17from pathlib import Path1819ROOT = Path(__file__).resolve().parents[1]20OUT = ROOT / "paper" / "sections" / "macros_stageE.tex"212223def load(p: str):24 with open(ROOT / p) as f:25 return json.load(f)262728def pct(x: float, d: int = 1) -> str:29 return f"{100 * x:.{d}f}"303132def signed(x: float, d: int = 1) -> str:33 return f"{x:+.{d}f}"343536def num(x: float, d: int = 2) -> str:37 return f"{x:.{d}f}"383940def thousands(n: int) -> str:41 return f"{int(n):,}".replace(",", "{,}")424344M: dict[str, str] = {}4546# ---------------------------------------------------------------- tokenizer / truncation (N1)47tok = load("results/n1/tokenizer_audit.json")48R, K = tok["frames"]["R"], tok["frames"]["K"]49M["mMaxSeq"] = str(tok["max_seq_length"])50M["mTokenizerMax"] = str(tok["model_max_length_tokenizer"])51M["mTruncShare"] = pct(R["share_truncated"])52M["mTruncShareFull"] = pct(R["share_truncated_full_512w"])53M["mTruncTokensMedian"] = str(int(R["tokens_median"]))54M["mTruncWindowWords"] = str(int(R["words_in_window_median"]))55M["mTruncMassDropped"] = pct(R["share_of_token_mass_dropped"], 0)56M["mTruncHitsBeyond"] = pct(K["share_hits_beyond_window"], 0)57M["mTruncHitChunksAllBeyond"] = pct(K["share_hit_chunks_all_hits_beyond"], 0)58M["mTruncScoreBeyond"] = num(K["mean_score_adopt_by_hit_class"]["beyond_only"], 2)59M["mTruncScoreInside"] = num(K["mean_score_adopt_by_hit_class"]["inside_only"], 2)60M["mTruncScoreBoth"] = num(K["mean_score_adopt_by_hit_class"]["both"], 2)61M["mTruncSampleN"] = thousands(R["n"])6263# ---------------------------------------------------------------- keyword baseline (N1)64kw = load("results/n1/keyword_baseline.json")65M["mKwFirmSpearman"] = num(kw["firm_year"]["adopt_vs_kw_core_wmean"]["spearman"], 2)66M["mKwShareZeroFirmYears"] = pct(kw["firm_year"]["adopt_vs_kw_core_wmean"]["share_zero_b"], 0)67M["mKwQfiveInKwQfive"] = pct(kw["firm_year"]["adopt_vs_kw_core_wmean"]["share_a_Q5_in_b_Q5"], 0)68sec = {r["sector"]: r for r in kw["by_sector"]}69M["mKwUtilSpearman"] = num(abs(sec["Utilities"]["spearman"]), 2)70M["mKwUtilShareZero"] = pct(sec["Utilities"]["share_zero_kw"], 0)71M["mKwUtilMeanZ"] = num(sec["Utilities"]["mean_z_compos"], 2)72M["mKwITMeanZ"] = num(sec["Information Technology"]["mean_z_compos"], 2)73M["mKwITSpearman"] = num(sec["Information Technology"]["spearman"], 2)74rank_by_z = sorted(kw["by_sector"], key=lambda r: -r["mean_z_compos"])75M["mUtilSectorRank"] = str([r["sector"] for r in rank_by_z].index("Utilities") + 1)76with open(ROOT / "results/n1/keyword_chunk_deciles.csv") as f:77 dec = {int(r["decile"]): r for r in csv.DictReader(f)}78M["mKwTopDecileShare"] = pct(float(dec[10]["share_core_hit"]))79M["mKwTopDecileShareExt"] = pct(float(dec[10]["share_ext_hit"]))80M["mKwBottomDecileShare"] = pct(float(dec[1]["share_core_hit"]), 2)81with open(ROOT / "results/n1/keyword_corr_by_year.csv") as f:82 rows = list(csv.DictReader(f))83early = [float(r["spearman_compos_kw"]) for r in rows if int(r["fiscal_year"]) <= 2016]84late = [float(r["spearman_compos_kw"]) for r in rows if int(r["fiscal_year"]) >= 2022]85M["mKwCorrEarlyMin"] = num(min(early), 2)86M["mKwCorrEarlyMax"] = num(max(early), 2)87M["mKwCorrLateMin"] = num(min(late), 2)88M["mKwCorrLateMax"] = num(max(late), 2)89M["mKwShareZeroTwentyTen"] = pct(float(rows[0]["share_zero_kw"]), 0)90M["mKwShareZeroTwentyTwentyFour"] = pct(float(rows[-1]["share_zero_kw"]), 0)9192# ---------------------------------------------------------------- alternative anchors, negation probe (N1, AR-2)93alt = load("results/n1/alt_anchor_results.json")94M["mStripRho"] = num(alt["firm_level_spearman_compos_vs_strip"], 3)95M["mCoreRho"] = num(alt["firm_level_spearman_compos_vs_core_strip"], 2)96M["mStripQfiveStays"] = pct(alt["firm_year"]["adopt_vs_adopt_strip"]["share_Q5_stays_Q5"], 0)97pn = alt["probe_negation"]["s_ai_adoption"]98M["mProbeAffirm"] = num(pn["mean_affirmative"], 3)99M["mProbeNegated"] = num(pn["mean_negated"], 3)100M["mProbeDiff"] = num(pn["diff"], 2)101M["mProbePairsShare"] = pct(pn["share_pairs_aff_gt_neg"], 0)102M["mProbeNegatedPct"] = pct(alt["probe_pct_rank_in_corpus"]["negated_ai"])103M["mProbeAffirmPct"] = pct(alt["probe_pct_rank_in_corpus"]["affirmative_ai"], 2)104M["mProbeUtilityPct"] = pct(alt["probe_pct_rank_in_corpus"]["utility_data_center_load"])105M["mProbeParaphrasePct"] = pct(alt["probe_pct_rank_in_corpus"]["paraphrase_no_keyword"])106M["mProbeCyberPct"] = pct(alt["probe_pct_rank_in_corpus"]["cyber_no_ai"])107M["mProbeUnrelatedPct"] = pct(alt["probe_pct_rank_in_corpus"]["unrelated"], 0)108gm = alt["probe_group_means"]109M["mProbeUtilInfra"] = num(gm["utility_data_center_load"]["s_ai_infrastructure"], 3)110M["mProbeUtilAdopt"] = num(gm["utility_data_center_load"]["s_ai_adoption"], 3)111M["mProbeParaphraseAdopt"] = num(gm["paraphrase_no_keyword"]["s_ai_adoption"], 3)112M["mProbeAutomationAdopt"] = num(gm["automation_no_ai"]["s_ai_adoption"], 3)113M["mProbeUnrelatedAdopt"] = num(gm["unrelated"]["s_ai_adoption"], 3)114q = alt["corpus_adopt_quantiles_50_90_95_99_999"]115M["mAdoptPninetynine"] = num(q[3], 2)116M["mAdoptPninetyninenine"] = num(q[4], 2)117ranks = {r["ticker"]: r for r in alt["ida_he_ranks"]}118M["mIdaRank"] = str(int(ranks["IDA"]["r"]))119M["mIdaRankStrip"] = str(int(ranks["IDA"]["rs"]))120M["mIdaRankCore"] = str(int(ranks["IDA"]["rc"]))121M["mHeRank"] = str(int(ranks["HE"]["r"]))122M["mHeRankStrip"] = str(int(ranks["HE"]["rs"]))123M["mHeRankCore"] = str(int(ranks["HE"]["rc"]))124M["mTopThirtyUtilOriginal"] = str(alt["top30_utilities_original"])125M["mTopThirtyUtilCore"] = str(alt["top30_utilities_core_strip"])126M["mTopThirtyOverlapStrip"] = str(alt["top30_overlap_strip"])127M["mSectorZUtil"] = num(alt["sector_mean_z"]["Utilities"]["z_compos"], 2)128M["mSectorZIT"] = num(alt["sector_mean_z"]["Information Technology"]["z_compos"], 2)129130# ---------------------------------------------------------------- construct validity rows (r3k_23)131con = load("results/r3k_v2/results_construct.json")132S = con["signals"]133134135def sig(key: str, prefix: str) -> None:136 s = S[key]137 if s.get("spearman_with_composite") is not None:138 M[prefix + "Rho"] = num(s["spearman_with_composite"], 2)139 M[prefix + "FullW"] = signed(100 * s["full_w"]["mean"])140 M[prefix + "FullWT"] = num(s["full_w"]["t"])141 M[prefix + "FullWTplain"] = num(s["full_w"]["t_plain"])142 M[prefix + "FMW"] = signed(s["fm_w"]["slope_pct"], 2)143 M[prefix + "FMWT"] = num(s["fm_w"]["t"])144 M[prefix + "PreW"] = signed(100 * s["pre_w"]["mean"])145 M[prefix + "PostW"] = signed(100 * s["post_w"]["mean"])146147148sig("z_KW", "mKw")149sig("z_KWlog", "mKwLog")150sig("z_ALT", "mStrip")151sig("z_ALTcore", "mCore")152sig("z_CYBERstrip", "mCyberStrip")153M["mKwShareZero"] = pct(S["z_KW"]["share_zero"])154hr = con["horse_race_w"]155M["mHrComposVsKw"] = signed(hr["composite_vs_keyword"]["z_AI_COMPOS"]["slope_pct"], 2)156M["mHrComposVsKwT"] = num(hr["composite_vs_keyword"]["z_AI_COMPOS"]["t"])157M["mHrKw"] = signed(hr["composite_vs_keyword"]["z_KW"]["slope_pct"], 2)158M["mHrKwT"] = num(hr["composite_vs_keyword"]["z_KW"]["t"])159M["mHrComposVsStrip"] = signed(hr["composite_vs_stripped"]["z_AI_COMPOS"]["slope_pct"], 2)160M["mHrComposVsStripT"] = num(hr["composite_vs_stripped"]["z_AI_COMPOS"]["t"])161M["mHrStrip"] = signed(hr["composite_vs_stripped"]["z_ALT"]["slope_pct"], 2)162M["mHrStripT"] = num(hr["composite_vs_stripped"]["z_ALT"]["t"])163exu = con["ex_utilities_energy"]164M["mExUtilFullW"] = signed(100 * exu["full_w"]["mean"])165M["mExUtilFullWT"] = num(exu["full_w"]["t"])166M["mExUtilN"] = thousands(exu["n"])167M["mUtilShareQfive"] = pct(con["utilities_share_q5_vs_all"][0])168M["mUtilShareAll"] = pct(con["utilities_share_q5_vs_all"][1])169M["mITShareQfive"] = pct(con["q5_sector_shares"]["Information Technology"], 0)170171# ---------------------------------------------------------------- survivorship bounding (r3k_18)172sv = load("results/r3k_v2/results_surv.json")173b = sv["bounding"]174req = b["required_exit_rate_difference_e1_minus_e5"]175M["mBoundExitThirty"] = pct(req["r_x=-0.3"])176M["mBoundExitHalf"] = pct(req["r_x=-0.5"])177M["mBoundExitFull"] = pct(req["r_x=-1.0"])178small = {k: v for k, v in b["measured_annualised_exit_small_tercile"].items() if k >= "2019"}179large = {k: v for k, v in b["measured_annualised_exit_large_tercile"].items() if k >= "2019"}180M["mExitSmallMin"] = pct(min(small.values()))181M["mExitSmallMax"] = pct(max(small.values()))182M["mExitLargeMin"] = pct(min(large.values()))183M["mExitLargeMax"] = pct(max(large.values()))184ratios = [small[k] / large[k] for k in small if k in large]185M["mExitRatioMin"] = num(min(ratios), 1)186M["mExitRatioMax"] = num(max(ratios), 1)187M["mnIWVsnapshotsSinceNineteen"] = str(len([k for k in sv["snapshots"] if str(k)[:4] >= "2019"]) if isinstance(sv["snapshots"], (list, dict)) else "")188tm = sv["then_members_present"]189if isinstance(tm, dict):190 shares = [v.get("share_members") for v in tm.values() if isinstance(v, dict) and v.get("share_members") is not None]191elif isinstance(tm, list):192 shares = [v.get("share_members") for v in tm if isinstance(v, dict) and v.get("share_members") is not None]193else:194 shares = []195if shares:196 M["mThenMembersMin"] = pct(min(shares), 0)197 M["mThenMembersMax"] = pct(max(shares), 0)198es = sv.get("entry_side", {})199if isinstance(es, dict) and "entrant_premium_w_pct" in es:200 M["mEntrantPremW"] = signed(es["entrant_premium_w_pct"])201202# ---------------------------------------------------------------- break placebos (r3k_16)203r2 = load("results/r3k_v2/results2.json")204pb = r2["placebo_breaks"]205cb = pb["permutation"]["contiguous_blocks"]206M["mContigAboveRaw"] = str(sum(1 for k, v in cb.items() if k != "2022" and v > cb["2022"]))207M["mContigN"] = str(len(cb) - 1)208cbw = pb["winsorized"]["permutation"]["contiguous_blocks"]209M["mContigAboveW"] = str(sum(1 for k, v in cbw.items() if k != "2022" and v > cbw["2022"]))210M["mContigBestYear"] = max((k for k in cb if k != "2022"), key=lambda k: cb[k])211M["mContigBestDiff"] = signed(100 * cb[M["mContigBestYear"]])212M["mPermSubsets"] = str(pb["permutation"]["n_subsets"])213M["mSupWaldCVfive"] = num(pb["sup_wald"]["cv05_simulated"], 1)214ck = r2["cluster_k"]["reversal"]215M["mRevWithWithoutTwelve"] = signed(ck["high_language_with_vs_without_projects"]["month_1_12_w"]["mean_pct"], 2)216M["mRevWithWithoutTwelveT"] = num(ck["high_language_with_vs_without_projects"]["month_1_12_w"]["t"])217M["mRevWithWithoutTwentyFour"] = signed(ck["high_language_with_vs_without_projects"]["month_13_24_w"]["mean_pct"], 2)218M["mRevWithWithoutTwentyFourT"] = num(ck["high_language_with_vs_without_projects"]["month_13_24_w"]["t"])219M["mRevNhighWithProjects"] = thousands(ck["high_language_with_vs_without_projects"]["n_high_language_with_projects"])220M["mRevNhigh"] = thousands(ck["high_language_with_vs_without_projects"]["n_high_language"])221M["mRevComposThirteenTwentyFourW"] = signed(100 * ck["Q5_minus_Q1_month_13_24"]["w"]["mean"])222M["mRevComposThirteenTwentyFourWT"] = num(ck["Q5_minus_Q1_month_13_24"]["w"]["t"])223ft = r2["filing_type"]224if "wald_equal_slopes_w" in ft:225 M["mFormWaldChi"] = num(ft["wald_equal_slopes_w"].get("stat", float("nan")), 2)226dt = r2["did_eventtime"]227M["mEvtRefSlope"] = signed(100 * dt["base_slope_ref_year"])228M["mEvtRefT"] = num(dt["base_t"])229rows_dt = {r["year"]: r for r in dt["rows"]}230M["mEvtDiffTwentyTwoT"] = num(rows_dt[2022]["diff_from_ref"] / rows_dt[2022]["diff_se"])231M["mEvtDiffTwentyT"] = num(rows_dt[2020]["diff_from_ref"] / rows_dt[2020]["diff_se"])232M["mEvtDiffTwentyEighteen"] = signed(100 * rows_dt[2018]["diff_from_ref"])233M["mEvtDiffTwenty"] = signed(100 * rows_dt[2020]["diff_from_ref"])234M["mEvtN"] = thousands(rows_dt[2022]["n"])235236# ---------------------------------------------------------------- market-cap diagnostics (r3k_14)237mc = load("results/r3k_v2/mktcap_diag.json")238M["mAaplCapVone"] = num(mc["AAPL_cohort2010_cap_v1_usd"] / 1e9, 1)239M["mAaplRankVone"] = str(int(mc["AAPL_cohort2010_rank_v1"]))240M["mAaplCapVtwo"] = str(int(round(mc["AAPL_cohort2010_cap_v2_usd"] / 1e9)))241M["mAaplRankVtwo"] = str(int(mc["AAPL_cohort2010_rank_v2"]))242M["mLogCapMaxVone"] = num(mc["log_cap_min_max_v1"][1], 1)243M["mAepCapVoneExp"] = str(int(math.floor(math.log10(mc["AEP_2010"]["cap_v1"]))))244M["mAepCapVoneMant"] = num(mc["AEP_2010"]["cap_v1"] / 10 ** math.floor(math.log10(mc["AEP_2010"]["cap_v1"])), 1)245M["mAepCapVtwo"] = num(mc["AEP_2010"]["cap_v2"] / 1e9, 1)246M["mnCapsGtThreeTrillionVone"] = str(mc["n_caps_gt_3e12_v1"])247M["mnCapsGtThreeTrillionVtwo"] = str(mc["n_caps_gt_3e12"])248M["mShareNoCapVone"] = pct(mc["share_est_without_cap_v1"])249M["mShareNoCapVtwo"] = pct(mc["share_est_without_cap_v2"])250M["mnSizeGroupChange"] = thousands(mc["n_firm_years_changing_size_group"])251252# ---------------------------------------------------------------- BEFORE values from the committed baseline (HEAD 3cf7560)253r1 = load("results/r3k/results.json")254r2v1 = load("results/r3k/results2.json")255ev1 = load("results/r3k/results_event.json")256M["mvoneNobs"] = thousands(r1["n_obs"])257M["mvoneNfirms"] = thousands(r1["n_firms"])258M["mvoneSpreadFull"] = signed(100 * r1["sorts"]["AI_COMPOS_EW"]["Q5_minus_Q1"])259M["mvoneSpreadFullT"] = num(r1["sorts"]["AI_COMPOS_EW"]["t_stat"])260M["mvoneSpreadFullVW"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q5_minus_Q1"])261M["mvoneSpreadFullVWT"] = num(r1["sorts"]["AI_COMPOS_VW"]["t_stat"])262M["mvoneVWQone"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q1"])263M["mvoneVWQfive"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q5"])264M["mvoneFMcompos"] = signed(100 * r1["fama_macbeth"]["compos"]["z_AI_COMPOS"]["mean_coef"], 2)265M["mvoneFMcomposT"] = num(r1["fama_macbeth"]["compos"]["z_AI_COMPOS"]["t"])266sc = r2v1["sector_controls"]["fm_full"]267M["mvoneFMsector"] = signed(sc["slope_sector_pct"], 2)268M["mvoneFMsectorT"] = num(sc["t_sector"])269M["mvoneSectorNeutralFull"] = signed(100 * r2v1["sector_neutral"]["full"]["mean"])270M["mvoneSectorNeutralFullT"] = num(r2v1["sector_neutral"]["full"]["t"])271M["mvoneSpreadPre"] = signed(r2v1["cohorts"]["pre_mean_pct"])272M["mvoneSpreadPost"] = signed(r2v1["cohorts"]["post_mean_pct"])273M["mvonePooledPost"] = signed(r2v1["cohorts"]["pooled_post"]["coef_pct"])274M["mvonePooledPostT"] = num(r2v1["cohorts"]["pooled_post"]["t"])275M["mvonePooledPostW"] = signed(r2v1["cohorts"]["pooled_post_wins"]["coef_pct"])276M["mvonePooledPostExTwentyFour"] = signed(r2v1["cohorts"]["pooled_post_excl_2024"]["coef_pct"])277M["mvonePooledPostExTwentyFourT"] = num(r2v1["cohorts"]["pooled_post_excl_2024"]["t"])278M["mvoneCohortTwentyTwo"] = signed(r2v1["cohorts"]["cohort_spreads_pct"]["2022"])279M["mvoneEntrantPrem"] = signed(r2v1["entrants"]["entrant_minus_seasoned_fwd_pct"])280g = r1["groups"]281M["mvoneLargePre"] = signed(100 * g["large_top1000"]["pre"]["mean"]) if "pre" in g["large_top1000"] else ""282M["mvoneLargePost"] = signed(100 * g["large_top1000"]["post"]["mean"]) if "post" in g["large_top1000"] else ""283M["mvoneSmallPre"] = signed(100 * g["small_rest"]["pre"]["mean"]) if "pre" in g["small_rest"] else ""284M["mvoneSmallPost"] = signed(100 * g["small_rest"]["post"]["mean"]) if "post" in g["small_rest"] else ""285M["mvoneEvFullHL"] = signed(ev1["car_1y"]["H_minus_L"])286M["mvoneEvFullWelch"] = num(ev1["car_1y"]["t_welch"])287M["mvoneEvLargeHL"] = signed(ev1["by_size"]["large"]["H_minus_L_1y"])288M["mvoneEvLargeWelch"] = num(ev1["by_size"]["large"]["t_welch"])289M["mvoneEvSmallHL"] = signed(ev1["by_size"]["small"]["H_minus_L_1y"])290M["mvoneEvSmallWelch"] = num(ev1["by_size"]["small"]["t_welch"])291M["mvoneEvSixtyHL"] = signed(ev1["car_60d"]["H_minus_L"])292M["mvoneEvN"] = thousands(ev1["n_firms"])293sp = r2v1["spid"]["z_SPID_TECH"]294M["mvoneSpidT"] = num(abs(sp["z_SPID_TECH:post"]["t"]))295M["mvoneSpidPostCoefExp"] = str(int(math.floor(math.log10(abs(sp["post"]["coef_pct"])))))296M["mvoneDidCont"] = signed(100 * r2v1["did"]["continuous"]["coef"], 2)297M["mvoneDidContT"] = num(r2v1["did"]["continuous"]["t"])298M["mvoneDidBin"] = signed(100 * r2v1["did"]["binary"]["coef"], 2)299M["mvoneDidBinT"] = num(r2v1["did"]["binary"]["t"])300evt1 = {r["year"]: r for r in r2v1["did_eventtime"]}301M["mvoneEvtTwentyOne"] = signed(100 * evt1[2021]["coef"])302M["mvoneEvtTwentyOneSE"] = num(100 * evt1[2021]["se"], 1)303M["mvoneEvtTwentyTwo"] = signed(100 * evt1[2022]["coef"])304M["mvoneEvtTwentyFour"] = signed(100 * evt1[2024]["coef"])305M["mvoneSpreadPostTnaive"] = num(r1["subperiod_spread"]["AI_COMPOS"]["post"]["t"]) if "post" in r1["subperiod_spread"]["AI_COMPOS"] else ""306M["mvoneSpidAnyPost"] = signed(r2v1["spid"]["spread_ai_any_post"]["post"]["mean_pct"])307M["mvoneSpidAnyPostT"] = num(r2v1["spid"]["spread_ai_any_post"]["post"]["t"])308M["mvoneSpidPre"] = signed(100 * r1["subperiod_spread"]["SPID_TECH"]["pre"]["mean"])309M["mvoneSpidPreT"] = num(r1["subperiod_spread"]["SPID_TECH"]["pre"]["t"])310M["mvoneSpidLSmean"] = signed(r1["ls_alpha"]["SPID_TECH"]["full"]["mean_ls_ann_pct"], 2)311M["mvonePlaceboRank"] = str(r2v1["placebo_breaks"]["rank_of_2022"]) if "placebo_breaks" in r2v1 else ""312313# ---------------------------------------------------------------- SPID extractor provenance (L8)314meta = load("results/r3k_v2/validation/spid_sample_meta.json")315M["mSpidModel"] = ", ".join(meta["extractor_provenance_from_code_snapshot"]["model"]).replace("_", "\\_")316M["mSpidSampleN"] = str(meta["n_sample"])317M["mSpidUniverseMentions"] = thousands(meta["n_universe_mentions"])318M["mSpidEvidenceShare"] = pct(meta["share_evidence_found"])319M["mSpidConfMedian"] = num(meta["confidence_quantiles"]["0.5"], 1)320M["mSpidDbDate"] = meta["db_mtime"][:10]321M["mSpidStrata"] = str(len(meta["strata_allocation"]))322323# ---------------------------------------------------------------- hygiene summary (r3k_13)324hy = load("results/r3k_v2/hygiene_summary.json")325M["mHygieneConfirmedMonthly"] = str(hy["verdicts_monthly"]["confirmed"])326M["mHygieneConfirmedFY"] = str(hy["verdicts_firm_year"]["confirmed"])327328329# ---------------------------------------------------------------- winsorized rows of results.json v2 (r3k_15)330rv = load("results/r3k_v2/results.json")331so = rv["sorts"]332M["mriskFullW"] = signed(100 * so["AI_RISK_EW_w"]["Q5_minus_Q1"])333M["mriskFullWT"] = num(so["AI_RISK_EW_w"]["t_stat"])334M["mriskFullWTplain"] = num(so["AI_RISK_EW_w"]["t_plain"])335M["mriskFull"] = signed(100 * so["AI_RISK_EW"]["Q5_minus_Q1"])336M["mriskFullT"] = num(so["AI_RISK_EW"]["t_stat"])337M["mriskTopOnePct"] = signed(100 * so["AI_RISK_EW"]["top1pct_contribution"])338M["mtenkFullW"] = signed(100 * so["AI_10k_EW_w"]["Q5_minus_Q1"])339M["mtenkFullWT"] = num(so["AI_10k_EW_w"]["t_stat"])340M["mformwFullWsort"] = signed(100 * so["AI_formw_EW_w"]["Q5_minus_Q1"])341M["mformwFullWsortT"] = num(so["AI_formw_EW_w"]["t_stat"])342M["mcyberFullW"] = signed(100 * so["CYBER_EW_w"]["Q5_minus_Q1"])343M["mcyberFullWT"] = num(so["CYBER_EW_w"]["t_stat"])344M["mcomposTopOnePctW"] = signed(100 * so["AI_COMPOS_EW_w"].get("top1pct_contribution", float("nan")))345sp = rv["subperiod_spread"]346for key, name in (("AI_10k", "tenk"), ("AI_RISK", "risk"), ("CYBER", "cyber"), ("AI_formw", "formw"), ("AI_PC1", "pca")):347 blk = sp[key]348 for per in ("pre", "post"):349 w = blk.get(per + "_w")350 if w:351 M[f"m{name}{per.capitalize()}Wsp"] = signed(100 * w["mean"])352 M[f"m{name}{per.capitalize()}WspT"] = num(w["t"])353g2 = rv["groups"]354for key, name in (("large_pct42", "largeFortyTwo"), ("small_pct42", "smallFiftyEight")):355 blk = g2[key]356 M[f"m{name}N"] = thousands(blk["n"])357 M[f"m{name}Firms"] = thousands(blk["firms"])358 for per in ("pre", "post", "full"):359 w = blk.get(per + "_w")360 if w:361 M[f"m{name}{per.capitalize()}W"] = signed(100 * w["mean"])362 M[f"m{name}{per.capitalize()}WT"] = num(w["t"])363 if "post_loo_w" in blk:364 M[f"m{name}PostExTwentyFourW"] = signed(100 * blk["post_loo_w"]["2024"])365ff = r2["filing_type"]["fm_forms_w"]["horse_race"]366for key, name in (("z_AI_10k", "TenK"), ("z_AI_10q", "TenQ"), ("z_AI_8k", "EightK")):367 if key in ff:368 M[f"mhr{name}W"] = signed(ff[key]["slope_pct"], 2)369 M[f"mhr{name}WT"] = num(ff[key]["t"])370en = r2["entrants"]371if "pre_w" in en["seasoned"]:372 M["mseasonedPreW"] = signed(100 * en["seasoned"]["pre_w"]["mean"])373 M["mseasonedPostW"] = signed(100 * en["seasoned"]["post_w"]["mean"]) if "post_w" in en["seasoned"] else ""374if "entrant_minus_seasoned_fwd_w_pct" in en:375 M["mentrantPremW"] = signed(en["entrant_minus_seasoned_fwd_w_pct"])376for k in ("entrant_minus_seasoned_fwd_pct",):377 if k in en: M["mentrantPremRaw"] = signed(en[k])378379# ---------------------------------------------------------------- cohort sizes, attrition 2019, 2021 split, contiguous block380nby = rv["n_by_year"]381M["mnObsFirstCohort"] = thousands(nby[min(nby)])382M["mnObsLastCohort"] = thousands(nby[max(nby)])383piv = sv["attrition_pivot_absent_share"]384M["mattritionSmallNineteen"] = pct(piv["20190531"]["small"], 0)385M["mattritionLargeNineteen"] = pct(piv["20190531"]["large"], 0)386M["mattritionSmallTwentyFour"] = pct(piv["20241206"]["small"], 0)387M["mattritionLargeTwentyFour"] = pct(piv["20241206"]["large"], 0)388cw = r2["cohorts_w"]389cs = cw.get("cohort_spreads_pct", cw)390M["mcohortTwentyOneW"] = signed(cs["2021"]) if isinstance(cs.get("2021"), (int, float)) else ""391br = r2["breaks"]392alt = br.get("split_2021")393if alt:394 M["mbreakTwentyOnePreW"] = signed(100 * alt["pre_w"]["mean"])395 M["mbreakTwentyOnePreWT"] = num(alt["pre_w"]["t"])396 M["mbreakTwentyOnePostW"] = signed(100 * alt["post_w"]["mean"])397 M["mbreakTwentyOnePostWT"] = num(alt["post_w"]["t"])398 pdw = alt["pooled_diff_w"]399 M["mbreakTwentyOneDiffW"] = signed(pdw["coef_pct"])400 M["mbreakTwentyOneDiffWT"] = num(pdw["t_firm"])401 M["mbreakTwentyOneDiffWTtw"] = num(pdw["t_twoway"])402by = M["mContigBestYear"]403M["mContigBestBlock"] = f"{by}--{int(by) + 2}"404405# ---------------------------------------------------------------- N1 corpus macros re-emitted with letter-only names406# results/n1/corpus_macros.tex (code/n1/n1_01_corpus_counts.py) uses digits in macro names (\mnFilings10K),407# which LaTeX cannot parse; the same values are re-emitted here as \mnFilingsTenK etc.408import re as _re409_digit_map = {"10K": "TenK", "10Q": "TenQ", "8K": "EightK"}410for line in (ROOT / "results/n1/corpus_macros.tex").read_text().splitlines():411 m = _re.match(r"\\newcommand\{\\(\w+)\}\{(.*)\}$", line.strip())412 if not m:413 continue414 name, val = m.group(1), m.group(2)415 for d, w in _digit_map.items():416 name = name.replace(d, w)417 if name not in M and name not in ("mnFilingsCorpus", "mnIssuersCorpus"):418 M[name] = val419420# ---------------------------------------------------------------- write421M = {k: v for k, v in M.items() if v not in ("", None)}422bad = [k for k in M if not k.isalpha()]423assert not bad, f"macro names must be letters only: {bad}"424# never redefine a macro that tables/macros.tex (Agent 9 / r3k_20) already provides: theirs wins425import re as _re2426_theirs = set(_re2.findall(r"\\newcommand\{\\(\w+)\}", (ROOT / "paper/tables/macros.tex").read_text()))427_dup = sorted(k for k in M if k in _theirs)428if _dup:429 print("skipping names already defined in tables/macros.tex:", _dup)430M = {k: v for k, v in M.items() if k not in _theirs}431lines = ["% Generated by scripts/stageE_writing_macros.py (Stage E, Agent 10) — do not edit by hand.",432 "% Sources: results/n1/*, results/r3k_v2/*, results/r3k/* (committed baseline 3cf7560 for the mvone* BEFORE values)."]433for k, v in M.items():434 lines.append(f"\\newcommand{{\\{k}}}{{{v}}}")435OUT.write_text("\n".join(lines) + "\n")436print(f"wrote {OUT.relative_to(ROOT)} with {len(M)} macros")437