|
1 |
+#!/usr/bin/env python3 |
|
2 |
+"""Stage E (Agent 10, writing) — supplementary number macros for the manuscript prose. |
|
3 |
+ |
|
4 |
+Every macro below is read from a result file produced by a Stage D / N1 script; nothing is typed. |
|
5 |
+Sources: results/n1/tokenizer_audit.json, keyword_baseline.json, keyword_chunk_deciles.csv, |
|
6 |
+keyword_corr_by_year.csv, alt_anchor_results.json; results/r3k_v2/results_construct.json, |
|
7 |
+results_surv.json, results2.json, mktcap_diag.json, validation/spid_sample_meta.json; |
|
8 |
+the committed baseline results/r3k/*.json (HEAD 3cf7560) for the BEFORE values. |
|
9 |
+Output: paper/sections/macros_stageE.tex (\\input from paper/main.tex after tables/macros.tex). |
|
10 |
+Usage: python3 -I scripts/stageE_writing_macros.py (run from the repository root) |
|
11 |
+""" |
|
12 |
+from __future__ import annotations |
|
13 |
+ |
|
14 |
+import csv |
|
15 |
+import json |
|
16 |
+import math |
|
17 |
+from pathlib import Path |
|
18 |
+ |
|
19 |
+ROOT = Path(__file__).resolve().parents[1] |
|
20 |
+OUT = ROOT / "paper" / "sections" / "macros_stageE.tex" |
|
21 |
+ |
|
22 |
+ |
|
23 |
+def load(p: str): |
|
24 |
+ with open(ROOT / p) as f: |
|
25 |
+ return json.load(f) |
|
26 |
+ |
|
27 |
+ |
|
28 |
+def pct(x: float, d: int = 1) -> str: |
|
29 |
+ return f"{100 * x:.{d}f}" |
|
30 |
+ |
|
31 |
+ |
|
32 |
+def signed(x: float, d: int = 1) -> str: |
|
33 |
+ return f"{x:+.{d}f}" |
|
34 |
+ |
|
35 |
+ |
|
36 |
+def num(x: float, d: int = 2) -> str: |
|
37 |
+ return f"{x:.{d}f}" |
|
38 |
+ |
|
39 |
+ |
|
40 |
+def thousands(n: int) -> str: |
|
41 |
+ return f"{int(n):,}".replace(",", "{,}") |
|
42 |
+ |
|
43 |
+ |
|
44 |
+M: dict[str, str] = {} |
|
45 |
+ |
|
46 |
+# ---------------------------------------------------------------- tokenizer / truncation (N1) |
|
47 |
+tok = load("results/n1/tokenizer_audit.json") |
|
48 |
+R, K = tok["frames"]["R"], tok["frames"]["K"] |
|
49 |
+M["mMaxSeq"] = str(tok["max_seq_length"]) |
|
50 |
+M["mTokenizerMax"] = str(tok["model_max_length_tokenizer"]) |
|
51 |
+M["mTruncShare"] = pct(R["share_truncated"]) |
|
52 |
+M["mTruncShareFull"] = pct(R["share_truncated_full_512w"]) |
|
53 |
+M["mTruncTokensMedian"] = str(int(R["tokens_median"])) |
|
54 |
+M["mTruncWindowWords"] = str(int(R["words_in_window_median"])) |
|
55 |
+M["mTruncMassDropped"] = pct(R["share_of_token_mass_dropped"], 0) |
|
56 |
+M["mTruncHitsBeyond"] = pct(K["share_hits_beyond_window"], 0) |
|
57 |
+M["mTruncHitChunksAllBeyond"] = pct(K["share_hit_chunks_all_hits_beyond"], 0) |
|
58 |
+M["mTruncScoreBeyond"] = num(K["mean_score_adopt_by_hit_class"]["beyond_only"], 2) |
|
59 |
+M["mTruncScoreInside"] = num(K["mean_score_adopt_by_hit_class"]["inside_only"], 2) |
|
60 |
+M["mTruncScoreBoth"] = num(K["mean_score_adopt_by_hit_class"]["both"], 2) |
|
61 |
+M["mTruncSampleN"] = thousands(R["n"]) |
|
62 |
+ |
|
63 |
+# ---------------------------------------------------------------- keyword baseline (N1) |
|
64 |
+kw = load("results/n1/keyword_baseline.json") |
|
65 |
+M["mKwFirmSpearman"] = num(kw["firm_year"]["adopt_vs_kw_core_wmean"]["spearman"], 2) |
|
66 |
+M["mKwShareZeroFirmYears"] = pct(kw["firm_year"]["adopt_vs_kw_core_wmean"]["share_zero_b"], 0) |
|
67 |
+M["mKwQfiveInKwQfive"] = pct(kw["firm_year"]["adopt_vs_kw_core_wmean"]["share_a_Q5_in_b_Q5"], 0) |
|
68 |
+sec = {r["sector"]: r for r in kw["by_sector"]} |
|
69 |
+M["mKwUtilSpearman"] = num(abs(sec["Utilities"]["spearman"]), 2) |
|
70 |
+M["mKwUtilShareZero"] = pct(sec["Utilities"]["share_zero_kw"], 0) |
|
71 |
+M["mKwUtilMeanZ"] = num(sec["Utilities"]["mean_z_compos"], 2) |
|
72 |
+M["mKwITMeanZ"] = num(sec["Information Technology"]["mean_z_compos"], 2) |
|
73 |
+M["mKwITSpearman"] = num(sec["Information Technology"]["spearman"], 2) |
|
74 |
+rank_by_z = sorted(kw["by_sector"], key=lambda r: -r["mean_z_compos"]) |
|
75 |
+M["mUtilSectorRank"] = str([r["sector"] for r in rank_by_z].index("Utilities") + 1) |
|
76 |
+with open(ROOT / "results/n1/keyword_chunk_deciles.csv") as f: |
|
77 |
+ dec = {int(r["decile"]): r for r in csv.DictReader(f)} |
|
78 |
+M["mKwTopDecileShare"] = pct(float(dec[10]["share_core_hit"])) |
|
79 |
+M["mKwTopDecileShareExt"] = pct(float(dec[10]["share_ext_hit"])) |
|
80 |
+M["mKwBottomDecileShare"] = pct(float(dec[1]["share_core_hit"]), 2) |
|
81 |
+with open(ROOT / "results/n1/keyword_corr_by_year.csv") as f: |
|
82 |
+ rows = list(csv.DictReader(f)) |
|
83 |
+early = [float(r["spearman_compos_kw"]) for r in rows if int(r["fiscal_year"]) <= 2016] |
|
84 |
+late = [float(r["spearman_compos_kw"]) for r in rows if int(r["fiscal_year"]) >= 2022] |
|
85 |
+M["mKwCorrEarlyMin"] = num(min(early), 2) |
|
86 |
+M["mKwCorrEarlyMax"] = num(max(early), 2) |
|
87 |
+M["mKwCorrLateMin"] = num(min(late), 2) |
|
88 |
+M["mKwCorrLateMax"] = num(max(late), 2) |
|
89 |
+M["mKwShareZeroTwentyTen"] = pct(float(rows[0]["share_zero_kw"]), 0) |
|
90 |
+M["mKwShareZeroTwentyTwentyFour"] = pct(float(rows[-1]["share_zero_kw"]), 0) |
|
91 |
+ |
|
92 |
+# ---------------------------------------------------------------- alternative anchors, negation probe (N1, AR-2) |
|
93 |
+alt = load("results/n1/alt_anchor_results.json") |
|
94 |
+M["mStripRho"] = num(alt["firm_level_spearman_compos_vs_strip"], 3) |
|
95 |
+M["mCoreRho"] = num(alt["firm_level_spearman_compos_vs_core_strip"], 2) |
|
96 |
+M["mStripQfiveStays"] = pct(alt["firm_year"]["adopt_vs_adopt_strip"]["share_Q5_stays_Q5"], 0) |
|
97 |
+pn = alt["probe_negation"]["s_ai_adoption"] |
|
98 |
+M["mProbeAffirm"] = num(pn["mean_affirmative"], 3) |
|
99 |
+M["mProbeNegated"] = num(pn["mean_negated"], 3) |
|
100 |
+M["mProbeDiff"] = num(pn["diff"], 2) |
|
101 |
+M["mProbePairsShare"] = pct(pn["share_pairs_aff_gt_neg"], 0) |
|
102 |
+M["mProbeNegatedPct"] = pct(alt["probe_pct_rank_in_corpus"]["negated_ai"]) |
|
103 |
+M["mProbeAffirmPct"] = pct(alt["probe_pct_rank_in_corpus"]["affirmative_ai"], 2) |
|
104 |
+M["mProbeUtilityPct"] = pct(alt["probe_pct_rank_in_corpus"]["utility_data_center_load"]) |
|
105 |
+M["mProbeParaphrasePct"] = pct(alt["probe_pct_rank_in_corpus"]["paraphrase_no_keyword"]) |
|
106 |
+M["mProbeCyberPct"] = pct(alt["probe_pct_rank_in_corpus"]["cyber_no_ai"]) |
|
107 |
+M["mProbeUnrelatedPct"] = pct(alt["probe_pct_rank_in_corpus"]["unrelated"], 0) |
|
108 |
+gm = alt["probe_group_means"] |
|
109 |
+M["mProbeUtilInfra"] = num(gm["utility_data_center_load"]["s_ai_infrastructure"], 3) |
|
110 |
+M["mProbeUtilAdopt"] = num(gm["utility_data_center_load"]["s_ai_adoption"], 3) |
|
111 |
+M["mProbeParaphraseAdopt"] = num(gm["paraphrase_no_keyword"]["s_ai_adoption"], 3) |
|
112 |
+M["mProbeAutomationAdopt"] = num(gm["automation_no_ai"]["s_ai_adoption"], 3) |
|
113 |
+M["mProbeUnrelatedAdopt"] = num(gm["unrelated"]["s_ai_adoption"], 3) |
|
114 |
+q = alt["corpus_adopt_quantiles_50_90_95_99_999"] |
|
115 |
+M["mAdoptPninetynine"] = num(q[3], 2) |
|
116 |
+M["mAdoptPninetyninenine"] = num(q[4], 2) |
|
117 |
+ranks = {r["ticker"]: r for r in alt["ida_he_ranks"]} |
|
118 |
+M["mIdaRank"] = str(int(ranks["IDA"]["r"])) |
|
119 |
+M["mIdaRankStrip"] = str(int(ranks["IDA"]["rs"])) |
|
120 |
+M["mIdaRankCore"] = str(int(ranks["IDA"]["rc"])) |
|
121 |
+M["mHeRank"] = str(int(ranks["HE"]["r"])) |
|
122 |
+M["mHeRankStrip"] = str(int(ranks["HE"]["rs"])) |
|
123 |
+M["mHeRankCore"] = str(int(ranks["HE"]["rc"])) |
|
124 |
+M["mTopThirtyUtilOriginal"] = str(alt["top30_utilities_original"]) |
|
125 |
+M["mTopThirtyUtilCore"] = str(alt["top30_utilities_core_strip"]) |
|
126 |
+M["mTopThirtyOverlapStrip"] = str(alt["top30_overlap_strip"]) |
|
127 |
+M["mSectorZUtil"] = num(alt["sector_mean_z"]["Utilities"]["z_compos"], 2) |
|
128 |
+M["mSectorZIT"] = num(alt["sector_mean_z"]["Information Technology"]["z_compos"], 2) |
|
129 |
+ |
|
130 |
+# ---------------------------------------------------------------- construct validity rows (r3k_23) |
|
131 |
+con = load("results/r3k_v2/results_construct.json") |
|
132 |
+S = con["signals"] |
|
133 |
+ |
|
134 |
+ |
|
135 |
+def sig(key: str, prefix: str) -> None: |
|
136 |
+ s = S[key] |
|
137 |
+ if s.get("spearman_with_composite") is not None: |
|
138 |
+ M[prefix + "Rho"] = num(s["spearman_with_composite"], 2) |
|
139 |
+ M[prefix + "FullW"] = signed(100 * s["full_w"]["mean"]) |
|
140 |
+ M[prefix + "FullWT"] = num(s["full_w"]["t"]) |
|
141 |
+ M[prefix + "FullWTplain"] = num(s["full_w"]["t_plain"]) |
|
142 |
+ M[prefix + "FMW"] = signed(s["fm_w"]["slope_pct"], 2) |
|
143 |
+ M[prefix + "FMWT"] = num(s["fm_w"]["t"]) |
|
144 |
+ M[prefix + "PreW"] = signed(100 * s["pre_w"]["mean"]) |
|
145 |
+ M[prefix + "PostW"] = signed(100 * s["post_w"]["mean"]) |
|
146 |
+ |
|
147 |
+ |
|
148 |
+sig("z_KW", "mKw") |
|
149 |
+sig("z_KWlog", "mKwLog") |
|
150 |
+sig("z_ALT", "mStrip") |
|
151 |
+sig("z_ALTcore", "mCore") |
|
152 |
+sig("z_CYBERstrip", "mCyberStrip") |
|
153 |
+M["mKwShareZero"] = pct(S["z_KW"]["share_zero"]) |
|
154 |
+hr = con["horse_race_w"] |
|
155 |
+M["mHrComposVsKw"] = signed(hr["composite_vs_keyword"]["z_AI_COMPOS"]["slope_pct"], 2) |
|
156 |
+M["mHrComposVsKwT"] = num(hr["composite_vs_keyword"]["z_AI_COMPOS"]["t"]) |
|
157 |
+M["mHrKw"] = signed(hr["composite_vs_keyword"]["z_KW"]["slope_pct"], 2) |
|
158 |
+M["mHrKwT"] = num(hr["composite_vs_keyword"]["z_KW"]["t"]) |
|
159 |
+M["mHrComposVsStrip"] = signed(hr["composite_vs_stripped"]["z_AI_COMPOS"]["slope_pct"], 2) |
|
160 |
+M["mHrComposVsStripT"] = num(hr["composite_vs_stripped"]["z_AI_COMPOS"]["t"]) |
|
161 |
+M["mHrStrip"] = signed(hr["composite_vs_stripped"]["z_ALT"]["slope_pct"], 2) |
|
162 |
+M["mHrStripT"] = num(hr["composite_vs_stripped"]["z_ALT"]["t"]) |
|
163 |
+exu = con["ex_utilities_energy"] |
|
164 |
+M["mExUtilFullW"] = signed(100 * exu["full_w"]["mean"]) |
|
165 |
+M["mExUtilFullWT"] = num(exu["full_w"]["t"]) |
|
166 |
+M["mExUtilN"] = thousands(exu["n"]) |
|
167 |
+M["mUtilShareQfive"] = pct(con["utilities_share_q5_vs_all"][0]) |
|
168 |
+M["mUtilShareAll"] = pct(con["utilities_share_q5_vs_all"][1]) |
|
169 |
+M["mITShareQfive"] = pct(con["q5_sector_shares"]["Information Technology"], 0) |
|
170 |
+ |
|
171 |
+# ---------------------------------------------------------------- survivorship bounding (r3k_18) |
|
172 |
+sv = load("results/r3k_v2/results_surv.json") |
|
173 |
+b = sv["bounding"] |
|
174 |
+req = b["required_exit_rate_difference_e1_minus_e5"] |
|
175 |
+M["mBoundExitThirty"] = pct(req["r_x=-0.3"]) |
|
176 |
+M["mBoundExitHalf"] = pct(req["r_x=-0.5"]) |
|
177 |
+M["mBoundExitFull"] = pct(req["r_x=-1.0"]) |
|
178 |
+small = {k: v for k, v in b["measured_annualised_exit_small_tercile"].items() if k >= "2019"} |
|
179 |
+large = {k: v for k, v in b["measured_annualised_exit_large_tercile"].items() if k >= "2019"} |
|
180 |
+M["mExitSmallMin"] = pct(min(small.values())) |
|
181 |
+M["mExitSmallMax"] = pct(max(small.values())) |
|
182 |
+M["mExitLargeMin"] = pct(min(large.values())) |
|
183 |
+M["mExitLargeMax"] = pct(max(large.values())) |
|
184 |
+ratios = [small[k] / large[k] for k in small if k in large] |
|
185 |
+M["mExitRatioMin"] = num(min(ratios), 1) |
|
186 |
+M["mExitRatioMax"] = num(max(ratios), 1) |
|
187 |
+M["mnIWVsnapshotsSinceNineteen"] = str(len([k for k in sv["snapshots"] if str(k)[:4] >= "2019"]) if isinstance(sv["snapshots"], (list, dict)) else "") |
|
188 |
+tm = sv["then_members_present"] |
|
189 |
+if isinstance(tm, dict): |
|
190 |
+ shares = [v.get("share_members") for v in tm.values() if isinstance(v, dict) and v.get("share_members") is not None] |
|
191 |
+elif isinstance(tm, list): |
|
192 |
+ shares = [v.get("share_members") for v in tm if isinstance(v, dict) and v.get("share_members") is not None] |
|
193 |
+else: |
|
194 |
+ shares = [] |
|
195 |
+if shares: |
|
196 |
+ M["mThenMembersMin"] = pct(min(shares), 0) |
|
197 |
+ M["mThenMembersMax"] = pct(max(shares), 0) |
|
198 |
+es = sv.get("entry_side", {}) |
|
199 |
+if isinstance(es, dict) and "entrant_premium_w_pct" in es: |
|
200 |
+ M["mEntrantPremW"] = signed(es["entrant_premium_w_pct"]) |
|
201 |
+ |
|
202 |
+# ---------------------------------------------------------------- break placebos (r3k_16) |
|
203 |
+r2 = load("results/r3k_v2/results2.json") |
|
204 |
+pb = r2["placebo_breaks"] |
|
205 |
+cb = pb["permutation"]["contiguous_blocks"] |
|
206 |
+M["mContigAboveRaw"] = str(sum(1 for k, v in cb.items() if k != "2022" and v > cb["2022"])) |
|
207 |
+M["mContigN"] = str(len(cb) - 1) |
|
208 |
+cbw = pb["winsorized"]["permutation"]["contiguous_blocks"] |
|
209 |
+M["mContigAboveW"] = str(sum(1 for k, v in cbw.items() if k != "2022" and v > cbw["2022"])) |
|
210 |
+M["mContigBestYear"] = max((k for k in cb if k != "2022"), key=lambda k: cb[k]) |
|
211 |
+M["mContigBestDiff"] = signed(100 * cb[M["mContigBestYear"]]) |
|
212 |
+M["mPermSubsets"] = str(pb["permutation"]["n_subsets"]) |
|
213 |
+M["mSupWaldCVfive"] = num(pb["sup_wald"]["cv05_simulated"], 1) |
|
214 |
+ck = r2["cluster_k"]["reversal"] |
|
215 |
+M["mRevWithWithoutTwelve"] = signed(ck["high_language_with_vs_without_projects"]["month_1_12_w"]["mean_pct"], 2) |
|
216 |
+M["mRevWithWithoutTwelveT"] = num(ck["high_language_with_vs_without_projects"]["month_1_12_w"]["t"]) |
|
217 |
+M["mRevWithWithoutTwentyFour"] = signed(ck["high_language_with_vs_without_projects"]["month_13_24_w"]["mean_pct"], 2) |
|
218 |
+M["mRevWithWithoutTwentyFourT"] = num(ck["high_language_with_vs_without_projects"]["month_13_24_w"]["t"]) |
|
219 |
+M["mRevNhighWithProjects"] = thousands(ck["high_language_with_vs_without_projects"]["n_high_language_with_projects"]) |
|
220 |
+M["mRevNhigh"] = thousands(ck["high_language_with_vs_without_projects"]["n_high_language"]) |
|
221 |
+M["mRevComposThirteenTwentyFourW"] = signed(100 * ck["Q5_minus_Q1_month_13_24"]["w"]["mean"]) |
|
222 |
+M["mRevComposThirteenTwentyFourWT"] = num(ck["Q5_minus_Q1_month_13_24"]["w"]["t"]) |
|
223 |
+ft = r2["filing_type"] |
|
224 |
+if "wald_equal_slopes_w" in ft: |
|
225 |
+ M["mFormWaldChi"] = num(ft["wald_equal_slopes_w"].get("stat", float("nan")), 2) |
|
226 |
+dt = r2["did_eventtime"] |
|
227 |
+M["mEvtRefSlope"] = signed(100 * dt["base_slope_ref_year"]) |
|
228 |
+M["mEvtRefT"] = num(dt["base_t"]) |
|
229 |
+rows_dt = {r["year"]: r for r in dt["rows"]} |
|
230 |
+M["mEvtDiffTwentyTwoT"] = num(rows_dt[2022]["diff_from_ref"] / rows_dt[2022]["diff_se"]) |
|
231 |
+M["mEvtDiffTwentyT"] = num(rows_dt[2020]["diff_from_ref"] / rows_dt[2020]["diff_se"]) |
|
232 |
+M["mEvtDiffTwentyEighteen"] = signed(100 * rows_dt[2018]["diff_from_ref"]) |
|
233 |
+M["mEvtDiffTwenty"] = signed(100 * rows_dt[2020]["diff_from_ref"]) |
|
234 |
+M["mEvtN"] = thousands(rows_dt[2022]["n"]) |
|
235 |
+ |
|
236 |
+# ---------------------------------------------------------------- market-cap diagnostics (r3k_14) |
|
237 |
+mc = load("results/r3k_v2/mktcap_diag.json") |
|
238 |
+M["mAaplCapVone"] = num(mc["AAPL_cohort2010_cap_v1_usd"] / 1e9, 1) |
|
239 |
+M["mAaplRankVone"] = str(int(mc["AAPL_cohort2010_rank_v1"])) |
|
240 |
+M["mAaplCapVtwo"] = str(int(round(mc["AAPL_cohort2010_cap_v2_usd"] / 1e9))) |
|
241 |
+M["mAaplRankVtwo"] = str(int(mc["AAPL_cohort2010_rank_v2"])) |
|
242 |
+M["mLogCapMaxVone"] = num(mc["log_cap_min_max_v1"][1], 1) |
|
243 |
+M["mAepCapVoneExp"] = str(int(math.floor(math.log10(mc["AEP_2010"]["cap_v1"])))) |
|
244 |
+M["mAepCapVoneMant"] = num(mc["AEP_2010"]["cap_v1"] / 10 ** math.floor(math.log10(mc["AEP_2010"]["cap_v1"])), 1) |
|
245 |
+M["mAepCapVtwo"] = num(mc["AEP_2010"]["cap_v2"] / 1e9, 1) |
|
246 |
+M["mnCapsGtThreeTrillionVone"] = str(mc["n_caps_gt_3e12_v1"]) |
|
247 |
+M["mnCapsGtThreeTrillionVtwo"] = str(mc["n_caps_gt_3e12"]) |
|
248 |
+M["mShareNoCapVone"] = pct(mc["share_est_without_cap_v1"]) |
|
249 |
+M["mShareNoCapVtwo"] = pct(mc["share_est_without_cap_v2"]) |
|
250 |
+M["mnSizeGroupChange"] = thousands(mc["n_firm_years_changing_size_group"]) |
|
251 |
+ |
|
252 |
+# ---------------------------------------------------------------- BEFORE values from the committed baseline (HEAD 3cf7560) |
|
253 |
+r1 = load("results/r3k/results.json") |
|
254 |
+r2v1 = load("results/r3k/results2.json") |
|
255 |
+ev1 = load("results/r3k/results_event.json") |
|
256 |
+M["mvoneNobs"] = thousands(r1["n_obs"]) |
|
257 |
+M["mvoneNfirms"] = thousands(r1["n_firms"]) |
|
258 |
+M["mvoneSpreadFull"] = signed(100 * r1["sorts"]["AI_COMPOS_EW"]["Q5_minus_Q1"]) |
|
259 |
+M["mvoneSpreadFullT"] = num(r1["sorts"]["AI_COMPOS_EW"]["t_stat"]) |
|
260 |
+M["mvoneSpreadFullVW"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q5_minus_Q1"]) |
|
261 |
+M["mvoneSpreadFullVWT"] = num(r1["sorts"]["AI_COMPOS_VW"]["t_stat"]) |
|
262 |
+M["mvoneVWQone"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q1"]) |
|
263 |
+M["mvoneVWQfive"] = signed(100 * r1["sorts"]["AI_COMPOS_VW"]["Q5"]) |
|
264 |
+M["mvoneFMcompos"] = signed(100 * r1["fama_macbeth"]["compos"]["z_AI_COMPOS"]["mean_coef"], 2) |
|
265 |
+M["mvoneFMcomposT"] = num(r1["fama_macbeth"]["compos"]["z_AI_COMPOS"]["t"]) |
|
266 |
+sc = r2v1["sector_controls"]["fm_full"] |
|
267 |
+M["mvoneFMsector"] = signed(sc["slope_sector_pct"], 2) |
|
268 |
+M["mvoneFMsectorT"] = num(sc["t_sector"]) |
|
269 |
+M["mvoneSectorNeutralFull"] = signed(100 * r2v1["sector_neutral"]["full"]["mean"]) |
|
270 |
+M["mvoneSectorNeutralFullT"] = num(r2v1["sector_neutral"]["full"]["t"]) |
|
271 |
+M["mvoneSpreadPre"] = signed(r2v1["cohorts"]["pre_mean_pct"]) |
|
272 |
+M["mvoneSpreadPost"] = signed(r2v1["cohorts"]["post_mean_pct"]) |
|
273 |
+M["mvonePooledPost"] = signed(r2v1["cohorts"]["pooled_post"]["coef_pct"]) |
|
274 |
+M["mvonePooledPostT"] = num(r2v1["cohorts"]["pooled_post"]["t"]) |
|
275 |
+M["mvonePooledPostW"] = signed(r2v1["cohorts"]["pooled_post_wins"]["coef_pct"]) |
|
276 |
+M["mvonePooledPostExTwentyFour"] = signed(r2v1["cohorts"]["pooled_post_excl_2024"]["coef_pct"]) |
|
277 |
+M["mvonePooledPostExTwentyFourT"] = num(r2v1["cohorts"]["pooled_post_excl_2024"]["t"]) |
|
278 |
+M["mvoneCohortTwentyTwo"] = signed(r2v1["cohorts"]["cohort_spreads_pct"]["2022"]) |
|
279 |
+M["mvoneEntrantPrem"] = signed(r2v1["entrants"]["entrant_minus_seasoned_fwd_pct"]) |
|
280 |
+g = r1["groups"] |
|
281 |
+M["mvoneLargePre"] = signed(100 * g["large_top1000"]["pre"]["mean"]) if "pre" in g["large_top1000"] else "" |
|
282 |
+M["mvoneLargePost"] = signed(100 * g["large_top1000"]["post"]["mean"]) if "post" in g["large_top1000"] else "" |
|
283 |
+M["mvoneSmallPre"] = signed(100 * g["small_rest"]["pre"]["mean"]) if "pre" in g["small_rest"] else "" |
|
284 |
+M["mvoneSmallPost"] = signed(100 * g["small_rest"]["post"]["mean"]) if "post" in g["small_rest"] else "" |
|
285 |
+M["mvoneEvFullHL"] = signed(ev1["car_1y"]["H_minus_L"]) |
|
286 |
+M["mvoneEvFullWelch"] = num(ev1["car_1y"]["t_welch"]) |
|
287 |
+M["mvoneEvLargeHL"] = signed(ev1["by_size"]["large"]["H_minus_L_1y"]) |
|
288 |
+M["mvoneEvLargeWelch"] = num(ev1["by_size"]["large"]["t_welch"]) |
|
289 |
+M["mvoneEvSmallHL"] = signed(ev1["by_size"]["small"]["H_minus_L_1y"]) |
|
290 |
+M["mvoneEvSmallWelch"] = num(ev1["by_size"]["small"]["t_welch"]) |
|
291 |
+M["mvoneEvSixtyHL"] = signed(ev1["car_60d"]["H_minus_L"]) |
|
292 |
+M["mvoneEvN"] = thousands(ev1["n_firms"]) |
|
293 |
+sp = r2v1["spid"]["z_SPID_TECH"] |
|
294 |
+M["mvoneSpidT"] = num(abs(sp["z_SPID_TECH:post"]["t"])) |
|
295 |
+M["mvoneSpidPostCoefExp"] = str(int(math.floor(math.log10(abs(sp["post"]["coef_pct"]))))) |
|
296 |
+M["mvoneDidCont"] = signed(100 * r2v1["did"]["continuous"]["coef"], 2) |
|
297 |
+M["mvoneDidContT"] = num(r2v1["did"]["continuous"]["t"]) |
|
298 |
+M["mvoneDidBin"] = signed(100 * r2v1["did"]["binary"]["coef"], 2) |
|
299 |
+M["mvoneDidBinT"] = num(r2v1["did"]["binary"]["t"]) |
|
300 |
+evt1 = {r["year"]: r for r in r2v1["did_eventtime"]} |
|
301 |
+M["mvoneEvtTwentyOne"] = signed(100 * evt1[2021]["coef"]) |
|
302 |
+M["mvoneEvtTwentyOneSE"] = num(100 * evt1[2021]["se"], 1) |
|
303 |
+M["mvoneEvtTwentyTwo"] = signed(100 * evt1[2022]["coef"]) |
|
304 |
+M["mvoneEvtTwentyFour"] = signed(100 * evt1[2024]["coef"]) |
|
305 |
+M["mvoneSpreadPostTnaive"] = num(r1["subperiod_spread"]["AI_COMPOS"]["post"]["t"]) if "post" in r1["subperiod_spread"]["AI_COMPOS"] else "" |
|
306 |
+M["mvoneSpidAnyPost"] = signed(r2v1["spid"]["spread_ai_any_post"]["post"]["mean_pct"]) |
|
307 |
+M["mvoneSpidAnyPostT"] = num(r2v1["spid"]["spread_ai_any_post"]["post"]["t"]) |
|
308 |
+M["mvoneSpidPre"] = signed(100 * r1["subperiod_spread"]["SPID_TECH"]["pre"]["mean"]) |
|
309 |
+M["mvoneSpidPreT"] = num(r1["subperiod_spread"]["SPID_TECH"]["pre"]["t"]) |
|
310 |
+M["mvoneSpidLSmean"] = signed(r1["ls_alpha"]["SPID_TECH"]["full"]["mean_ls_ann_pct"], 2) |
|
311 |
+M["mvonePlaceboRank"] = str(r2v1["placebo_breaks"]["rank_of_2022"]) if "placebo_breaks" in r2v1 else "" |
|
312 |
+ |
|
313 |
+# ---------------------------------------------------------------- SPID extractor provenance (L8) |
|
314 |
+meta = load("results/r3k_v2/validation/spid_sample_meta.json") |
|
315 |
+M["mSpidModel"] = ", ".join(meta["extractor_provenance_from_code_snapshot"]["model"]).replace("_", "\\_") |
|
316 |
+M["mSpidSampleN"] = str(meta["n_sample"]) |
|
317 |
+M["mSpidUniverseMentions"] = thousands(meta["n_universe_mentions"]) |
|
318 |
+M["mSpidEvidenceShare"] = pct(meta["share_evidence_found"]) |
|
319 |
+M["mSpidConfMedian"] = num(meta["confidence_quantiles"]["0.5"], 1) |
|
320 |
+M["mSpidDbDate"] = meta["db_mtime"][:10] |
|
321 |
+M["mSpidStrata"] = str(len(meta["strata_allocation"])) |
|
322 |
+ |
|
323 |
+# ---------------------------------------------------------------- hygiene summary (r3k_13) |
|
324 |
+hy = load("results/r3k_v2/hygiene_summary.json") |
|
325 |
+M["mHygieneConfirmedMonthly"] = str(hy["verdicts_monthly"]["confirmed"]) |
|
326 |
+M["mHygieneConfirmedFY"] = str(hy["verdicts_firm_year"]["confirmed"]) |
|
327 |
+ |
|
328 |
+ |
|
329 |
+# ---------------------------------------------------------------- winsorized rows of results.json v2 (r3k_15) |
|
330 |
+rv = load("results/r3k_v2/results.json") |
|
331 |
+so = rv["sorts"] |
|
332 |
+M["mriskFullW"] = signed(100 * so["AI_RISK_EW_w"]["Q5_minus_Q1"]) |
|
333 |
+M["mriskFullWT"] = num(so["AI_RISK_EW_w"]["t_stat"]) |
|
334 |
+M["mriskFullWTplain"] = num(so["AI_RISK_EW_w"]["t_plain"]) |
|
335 |
+M["mriskFull"] = signed(100 * so["AI_RISK_EW"]["Q5_minus_Q1"]) |
|
336 |
+M["mriskFullT"] = num(so["AI_RISK_EW"]["t_stat"]) |
|
337 |
+M["mriskTopOnePct"] = signed(100 * so["AI_RISK_EW"]["top1pct_contribution"]) |
|
338 |
+M["mtenkFullW"] = signed(100 * so["AI_10k_EW_w"]["Q5_minus_Q1"]) |
|
339 |
+M["mtenkFullWT"] = num(so["AI_10k_EW_w"]["t_stat"]) |
|
340 |
+M["mformwFullWsort"] = signed(100 * so["AI_formw_EW_w"]["Q5_minus_Q1"]) |
|
341 |
+M["mformwFullWsortT"] = num(so["AI_formw_EW_w"]["t_stat"]) |
|
342 |
+M["mcyberFullW"] = signed(100 * so["CYBER_EW_w"]["Q5_minus_Q1"]) |
|
343 |
+M["mcyberFullWT"] = num(so["CYBER_EW_w"]["t_stat"]) |
|
344 |
+M["mcomposTopOnePctW"] = signed(100 * so["AI_COMPOS_EW_w"].get("top1pct_contribution", float("nan"))) |
|
345 |
+sp = rv["subperiod_spread"] |
|
346 |
+for key, name in (("AI_10k", "tenk"), ("AI_RISK", "risk"), ("CYBER", "cyber"), ("AI_formw", "formw"), ("AI_PC1", "pca")): |
|
347 |
+ blk = sp[key] |
|
348 |
+ for per in ("pre", "post"): |
|
349 |
+ w = blk.get(per + "_w") |
|
350 |
+ if w: |
|
351 |
+ M[f"m{name}{per.capitalize()}Wsp"] = signed(100 * w["mean"]) |
|
352 |
+ M[f"m{name}{per.capitalize()}WspT"] = num(w["t"]) |
|
353 |
+g2 = rv["groups"] |
|
354 |
+for key, name in (("large_pct42", "largeFortyTwo"), ("small_pct42", "smallFiftyEight")): |
|
355 |
+ blk = g2[key] |
|
356 |
+ M[f"m{name}N"] = thousands(blk["n"]) |
|
357 |
+ M[f"m{name}Firms"] = thousands(blk["firms"]) |
|
358 |
+ for per in ("pre", "post", "full"): |
|
359 |
+ w = blk.get(per + "_w") |
|
360 |
+ if w: |
|
361 |
+ M[f"m{name}{per.capitalize()}W"] = signed(100 * w["mean"]) |
|
362 |
+ M[f"m{name}{per.capitalize()}WT"] = num(w["t"]) |
|
363 |
+ if "post_loo_w" in blk: |
|
364 |
+ M[f"m{name}PostExTwentyFourW"] = signed(100 * blk["post_loo_w"]["2024"]) |
|
365 |
+ff = r2["filing_type"]["fm_forms_w"]["horse_race"] |
|
366 |
+for key, name in (("z_AI_10k", "TenK"), ("z_AI_10q", "TenQ"), ("z_AI_8k", "EightK")): |
|
367 |
+ if key in ff: |
|
368 |
+ M[f"mhr{name}W"] = signed(ff[key]["slope_pct"], 2) |
|
369 |
+ M[f"mhr{name}WT"] = num(ff[key]["t"]) |
|
370 |
+en = r2["entrants"] |
|
371 |
+if "pre_w" in en["seasoned"]: |
|
372 |
+ M["mseasonedPreW"] = signed(100 * en["seasoned"]["pre_w"]["mean"]) |
|
373 |
+ M["mseasonedPostW"] = signed(100 * en["seasoned"]["post_w"]["mean"]) if "post_w" in en["seasoned"] else "" |
|
374 |
+if "entrant_minus_seasoned_fwd_w_pct" in en: |
|
375 |
+ M["mentrantPremW"] = signed(en["entrant_minus_seasoned_fwd_w_pct"]) |
|
376 |
+for k in ("entrant_minus_seasoned_fwd_pct",): |
|
377 |
+ if k in en: M["mentrantPremRaw"] = signed(en[k]) |
|
378 |
+ |
|
379 |
+# ---------------------------------------------------------------- cohort sizes, attrition 2019, 2021 split, contiguous block |
|
380 |
+nby = rv["n_by_year"] |
|
381 |
+M["mnObsFirstCohort"] = thousands(nby[min(nby)]) |
|
382 |
+M["mnObsLastCohort"] = thousands(nby[max(nby)]) |
|
383 |
+piv = sv["attrition_pivot_absent_share"] |
|
384 |
+M["mattritionSmallNineteen"] = pct(piv["20190531"]["small"], 0) |
|
385 |
+M["mattritionLargeNineteen"] = pct(piv["20190531"]["large"], 0) |
|
386 |
+M["mattritionSmallTwentyFour"] = pct(piv["20241206"]["small"], 0) |
|
387 |
+M["mattritionLargeTwentyFour"] = pct(piv["20241206"]["large"], 0) |
|
388 |
+cw = r2["cohorts_w"] |
|
389 |
+cs = cw.get("cohort_spreads_pct", cw) |
|
390 |
+M["mcohortTwentyOneW"] = signed(cs["2021"]) if isinstance(cs.get("2021"), (int, float)) else "" |
|
391 |
+br = r2["breaks"] |
|
392 |
+alt = br.get("split_2021") |
|
393 |
+if alt: |
|
394 |
+ M["mbreakTwentyOnePreW"] = signed(100 * alt["pre_w"]["mean"]) |
|
395 |
+ M["mbreakTwentyOnePreWT"] = num(alt["pre_w"]["t"]) |
|
396 |
+ M["mbreakTwentyOnePostW"] = signed(100 * alt["post_w"]["mean"]) |
|
397 |
+ M["mbreakTwentyOnePostWT"] = num(alt["post_w"]["t"]) |
|
398 |
+ pdw = alt["pooled_diff_w"] |
|
399 |
+ M["mbreakTwentyOneDiffW"] = signed(pdw["coef_pct"]) |
|
400 |
+ M["mbreakTwentyOneDiffWT"] = num(pdw["t_firm"]) |
|
401 |
+ M["mbreakTwentyOneDiffWTtw"] = num(pdw["t_twoway"]) |
|
402 |
+by = M["mContigBestYear"] |
|
403 |
+M["mContigBestBlock"] = f"{by}--{int(by) + 2}" |
|
404 |
+ |
|
405 |
+# ---------------------------------------------------------------- N1 corpus macros re-emitted with letter-only names |
|
406 |
+# results/n1/corpus_macros.tex (code/n1/n1_01_corpus_counts.py) uses digits in macro names (\mnFilings10K), |
|
407 |
+# which LaTeX cannot parse; the same values are re-emitted here as \mnFilingsTenK etc. |
|
408 |
+import re as _re |
|
409 |
+_digit_map = {"10K": "TenK", "10Q": "TenQ", "8K": "EightK"} |
|
410 |
+for line in (ROOT / "results/n1/corpus_macros.tex").read_text().splitlines(): |
|
411 |
+ m = _re.match(r"\\newcommand\{\\(\w+)\}\{(.*)\}$", line.strip()) |
|
412 |
+ if not m: |
|
413 |
+ continue |
|
414 |
+ name, val = m.group(1), m.group(2) |
|
415 |
+ for d, w in _digit_map.items(): |
|
416 |
+ name = name.replace(d, w) |
|
417 |
+ if name not in M and name not in ("mnFilingsCorpus", "mnIssuersCorpus"): |
|
418 |
+ M[name] = val |
|
419 |
+ |
|
420 |
+# ---------------------------------------------------------------- write |
|
421 |
+M = {k: v for k, v in M.items() if v not in ("", None)} |
|
422 |
+bad = [k for k in M if not k.isalpha()] |
|
423 |
+assert not bad, f"macro names must be letters only: {bad}" |
|
424 |
+# never redefine a macro that tables/macros.tex (Agent 9 / r3k_20) already provides: theirs wins |
|
425 |
+import re as _re2 |
|
426 |
+_theirs = set(_re2.findall(r"\\newcommand\{\\(\w+)\}", (ROOT / "paper/tables/macros.tex").read_text())) |
|
427 |
+_dup = sorted(k for k in M if k in _theirs) |
|
428 |
+if _dup: |
|
429 |
+ print("skipping names already defined in tables/macros.tex:", _dup) |
|
430 |
+M = {k: v for k, v in M.items() if k not in _theirs} |
|
431 |
+lines = ["% Generated by scripts/stageE_writing_macros.py (Stage E, Agent 10) — do not edit by hand.", |
|
432 |
+ "% Sources: results/n1/*, results/r3k_v2/*, results/r3k/* (committed baseline 3cf7560 for the mvone* BEFORE values)."] |
|
433 |
+for k, v in M.items(): |
|
434 |
+ lines.append(f"\\newcommand{{\\{k}}}{{{v}}}") |
|
435 |
+OUT.write_text("\n".join(lines) + "\n") |
|
436 |
+print(f"wrote {OUT.relative_to(ROOT)} with {len(M)} macros") |
|
437 |
|