1#!/usr/bin/env python2"""Phase 6: generate LaTeX tables, figures and a results-macro file from saved artifacts (never from memory)."""3from __future__ import annotations45import json6import sys7from pathlib import Path89import numpy as np10import pandas as pd11import polars as pl1213sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))14from wp39 import figures as F # noqa: E40215from wp39.pipeline import Paths, get_logger, load_config # noqa: E4021617T = {} # macros181920def macro(name: str, value, fmt: str = "{}"):21 T[name] = fmt.format(value)222324def fnum(x, d=4):25 return "" if x is None or (isinstance(x, float) and np.isnan(x)) else f"{x:.{d}f}"262728def stars(p):29 return "***" if p < 0.01 else "**" if p < 0.05 else "*" if p < 0.1 else ""303132def tex_table(df: pd.DataFrame, path: Path, cols: list[tuple[str, str, str]], note: str = "", group: str | None = None):33 """cols: (column, header, fmt) — fmt is a python format or 'int'/'str'/'tex'. `group`: column whose change inserts a rule."""34 lines = ["\\begin{tabular}{l" + "r" * (len(cols) - 1) + "}", "\\toprule", " & ".join(h for _, h, _ in cols) + " \\\\", "\\midrule"]35 prev = None36 for _, r in df.iterrows():37 if group is not None:38 g = r[group] if group in df.columns else None39 if prev is not None and g != prev:40 lines.append("\\addlinespace[2pt]\\midrule")41 prev = g42 cells = []43 for c, _, fmt in cols:44 v = r[c]45 if fmt == "int":46 cells.append(f"{int(v):,}" if pd.notna(v) else "")47 elif fmt == "str":48 cells.append(str(v).replace("_", " ").replace("%", "\\%"))49 elif fmt == "tex":50 cells.append(str(v))51 else:52 cells.append(fmt.format(v) if pd.notna(v) else "")53 lines.append(" & ".join(cells) + " \\\\")54 lines += ["\\bottomrule", "\\end{tabular}"]55 path.write_text("\n".join(lines) + "\n")565758def main() -> int:59 P = Paths.default()60 log = get_logger("09_manuscript", P.logs / "09_manuscript.log")61 cfg = load_config("research"); hp = cfg["cutoffs"]["primary_horizon_td"]62 E, R, TB, FG = P.evaluation, P.results, P.paper_tables, P.paper_figures63 ua = json.load(open(R / "universe_audit.json")); pa = json.load(open(R / "pit_panel_audit.json")); sa = json.load(open(R / "sample_audit.json"))64 ev = pd.read_parquet(P.processed / "sample_event.parquet", columns=["entity_id", "fiscal_period_end", "h", "cutoff", "A_q", "y_adverse_opinc",65 "S_opinc", "f_log_mcap", "wd_amihud60", "f_leverage", "release_time", "sic_division"])66 e20 = ev[ev.h == hp]67 # ---------------- macros: sample68 macro("nEntities", ua["entities"], "{:,}"); macro("nEligible", ua["eligible"], "{:,}"); macro("nWithCik", ua["with_cik"], "{:,}")69 macro("nFirmQuarters", pa["firm_quarters"], "{:,}"); macro("nFirmsPanel", pa["entities"], "{:,}")70 macro("nSampleEvent", sa["event"]["rows"], "{:,}"); macro("nSampleFirms", sa["event"]["firms"], "{:,}")71 macro("nSampleQuarters", sa["event"]["quarters"], "{:,}"); macro("nSampleCalendar", sa["calendar"]["rows"], "{:,}")72 macro("nObsPrimary", len(e20), "{:,}"); macro("baseRatePrimary", 100 * e20.y_adverse_opinc.mean(), "{:.1f}")73 macro("firstCutoff", str(e20.cutoff.min())[:10]); macro("lastCutoff", str(e20.cutoff.max())[:10])74 macro("releaseMatchedShare", 100 * next(r["len"] for r in pa["release_match"] if r["release_match"] == "ok") / pa["firm_quarters"], "{:.1f}")75 macro("filingLagMedian", pa["filing_minus_release_days_quantiles"]["0.5"], "{:.0f}")76 macro("epsEstimateShare", 100 * pa["eps_estimate_available_share"], "{:.0f}")77 # ---------------- Table 2: universe cascade78 tex_table(pd.read_csv(R / "table_universe_cascade.csv"), TB / "tab02_universe.tex", [("step", "Step", "str"), ("n", "Firms", "int")])79 casc = pd.read_csv(R / "table_sample_cascade_event.csv")80 tex_table(casc, TB / "tab02b_sample_cascade.tex", [("step", "Exclusion step", "str"), ("remaining", "Remaining", "int"), ("dropped", "Dropped", "int")])81 # ---------------- Table 3: summary statistics (h = 20 sample)82 feat = pd.read_parquet(P.processed / "sample_event.parquet")83 feat = feat[feat.h == hp]84 desc_cols = {"S_opinc": "Scaled operating-income surprise $S_q$", "y_adverse_opinc": "Adverse surprise ($S_q \\le -0.5\\%$)", "f_roa": "ROA (latest available)",85 "f_leverage": "Debt / assets", "f_log_mcap": "log market cap", "wd_ret20": "20-day return", "wd_vol20": "20-day daily volatility",86 "wd_amihud60": "Amihud illiquidity (60d, $\\times 10^6$)", "w1m_rv": "1-min realized variance (daily mean)", "w1m_rj": "1-min relative jump",87 "w1m_rsneg_share": "1-min negative semivariance share", "w1m_down_vol_share": "1-min down-bar volume share", "w1m_present_share": "1-min bars present (share)",88 "w30m_rv": "30-min realized variance", "w5m_rv": "5-min realized variance", "info_age_td": "Days since last release"}89 rows = []90 for c, lab in desc_cols.items():91 s = pd.to_numeric(feat[c], errors="coerce").replace([np.inf, -np.inf], np.nan).dropna()92 rows.append({"var": lab, "n": len(s), "mean": s.mean(), "sd": s.std(), "p10": s.quantile(0.1), "p50": s.median(), "p90": s.quantile(0.9)})93 tex_table(pd.DataFrame(rows), TB / "tab03_summary.tex", [("var", "Variable", "tex"), ("n", "N", "int"), ("mean", "Mean", "{:.4g}"), ("sd", "SD", "{:.4g}"),94 ("p10", "P10", "{:.4g}"), ("p50", "Median", "{:.4g}"), ("p90", "P90", "{:.4g}")])95 # ---------------- Table 4: information sets96 fc = json.load(open(E / "feature_columns_event.json"))97 info = pd.DataFrame([{"set": "B1", "desc": "PIT fundamentals + history + size", "k": len(fc["fundamentals"])},98 {"set": "B2", "desc": "B1 + daily OHLCV (20/60-day)", "k": len(fc["fundamentals"]) + len(fc["daily"])},99 {"set": "B3", "desc": "B2 + 30-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_30m"])},100 {"set": "B4", "desc": "B2 + 5-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_5m"])},101 {"set": "B5", "desc": "B2 + 1-minute families", "k": len(fc["fundamentals"]) + len(fc["daily"]) + len(fc["intraday_1m"])},102 {"set": "B5all", "desc": "B2 + 30m + 5m + 1m", "k": sum(len(v) for v in fc.values())}])103 tex_table(info, TB / "tab04_info_sets.tex", [("set", "Set", "str"), ("desc", "Information set", "str"), ("k", "Features", "int")])104 macro("kFund", len(fc["fundamentals"])); macro("kDaily", len(fc["daily"])); macro("kIntra", len(fc["intraday_1m"]))105 # ---------------- Table 5: headline OOS metrics (h=20, chrono test)106 met = pd.read_csv(E / "metrics_all.csv")107 m20 = met[(met.h == hp) & (met.target == "y_adverse_opinc") & (met.part == "test") & (met.tag == "chrono")].copy()108 order = ["B0", "B0b", "B1", "B2", "B3", "B4", "B5", "B5all", "LB1", "LB2", "LB3", "LB4", "LB5", "LB5all"]109 m20["o"] = m20.model.map({m: i for i, m in enumerate(order)}); m20 = m20.sort_values("o")110 m20["fam"] = np.where(m20.model.str.startswith("L"), "lightgbm", np.where(m20.model.str.startswith("B0"), "base", "logistic"))111 tex_table(m20, TB / "tab05_headline.tex", group="fam", cols= [("model", "Model", "str"), ("n", "N", "int"), ("logloss", "Log loss", "{:.4f}"), ("brier", "Brier", "{:.4f}"),112 ("auc", "AUC", "{:.3f}"), ("pr_auc", "PR-AUC", "{:.3f}"), ("cal_slope", "Cal. slope", "{:.2f}"), ("cal_intercept", "Cal. int.", "{:.2f}")])113 for _, r in m20.iterrows():114 macro(f"ll{r.model}", r.logloss, "{:.4f}"); macro(f"auc{r.model}", r.auc, "{:.3f}")115 # ---------------- Table 5b: paired contrasts116 con = pd.read_csv(E / "contrasts.csv")117 con["ci"] = con.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)118 con["gwp"] = con.apply(lambda r: f"{r.gw_p:.3f}{stars(r.gw_p)}", axis=1)119 con["holm"] = con["gw_p_holm"].map(lambda v: "" if pd.isna(v) else f"{v:.3f}")120 tex_table(con, TB / "tab05b_contrasts.tex", group="family", cols=[("family", "Family", "str"), ("a", "A", "str"), ("b", "B", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"),121 ("ci", "95\\% CI", "str"), ("gwp", "GW $p$", "str"), ("holm", "Holm", "str"), ("d_brier", "$\\Delta$ Brier", "{:+.5f}")])122 for _, r in con.iterrows():123 k = f"{r.family[:3]}{r.a}vs{r.b}".replace("B5all", "Ball")124 macro(f"d{k}", r.d_logloss, "{:+.4f}"); macro(f"lo{k}", r.ci_lo, "{:+.4f}"); macro(f"hi{k}", r.ci_hi, "{:+.4f}"); macro(f"p{k}", r.gw_p, "{:.3f}")125 macro(f"n{k}", r.n, "{:,}")126 # ---------------- Table 7: horizon ladder, secondary targets, firm holdout127 lad = pd.read_csv(E / "horizon_ladder.csv")128 lad["ci"] = lad.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)129 tex_table(lad.sort_values(["family", "h"], ascending=[True, False]), TB / "tab07_horizons.tex", group="family", cols=[("family", "Family", "str"), ("h", "$h$", "int"), ("n", "N", "int"),130 ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])131 h4 = json.load(open(E / "h4_timing.json"))132 for fam, v in h4.items():133 macro(f"h4rho{fam}", v["spearman_gain_vs_minus_h"], "{:+.2f}"); macro(f"h4p{fam}", v["p"], "{:.2f}")134 sec = pd.read_csv(E / "secondary_targets.csv")135 sec["ci"] = sec.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)136 tex_table(sec, TB / "tab10_targets.tex", group="target", cols=[("target", "Target", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"),137 ("auc_b", "AUC B2", "{:.3f}"), ("auc_a", "AUC B5", "{:.3f}"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])138 fh = pd.read_csv(E / "firm_holdout.csv"); fh["ci"] = fh.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)139 for _, r in fh.iterrows():140 macro(f"d{r.family[:3]}{r.tag.replace('_', '')}", r.d_logloss, "{:+.4f}"); macro(f"lo{r.family[:3]}{r.tag.replace('_', '')}", r.ci_lo, "{:+.4f}"); macro(f"hi{r.family[:3]}{r.tag.replace('_', '')}", r.ci_hi, "{:+.4f}")141 tex_table(fh, TB / "tab09b_firmholdout.tex", [("tag", "Firms", "str"), ("family", "Family", "str"), ("n", "N", "int"), ("d_logloss", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str"), ("gw_p", "GW $p$", "{:.3f}")])142 # ---------------- Table 6: panel regressions143 pr = pd.read_csv(E / "panel_regressions.csv")144 pr = pr[pr["var"].isin(["lfds5", "lfds2", "d_lfds", "dlfds_x_size0", "dlfds_x_size2", "dlfds_x_illiq0", "dlfds_x_illiq2", "dlfds_x_lev0", "dlfds_x_lev2"])].copy()145 pr["coef_s"] = pr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); pr["se_s"] = pr.se.map(lambda v: f"({v:.4f})")146 tex_table(pr, TB / "tab06_panel.tex", group="spec", cols=[("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int"), ("n_firms", "Firms", "int"), ("n_quarters", "Quarters", "int")])147 for _, r in pr.iterrows():148 macro(f"pan{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"pant{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}")149 # ---------------- Table 7b: mechanisms / subgroups; Table 8: placebos150 sub = pd.read_csv(E / "subgroups.csv"); sub["ci"] = sub.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)151 tex_table(sub[sub.family == "lightgbm"], TB / "tab07b_subgroups.tex", [("subgroup", "Subgroup", "str"), ("n", "N", "int"), ("base_rate", "Base rate", "{:.3f}"),152 ("auc_b", "AUC LB2", "{:.3f}"), ("auc_a", "AUC LB5", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")])153 plc = pd.read_csv(E / "placebo_shuffled.csv"); plc["ci"] = plc.apply(lambda r: f"[{r.ci_lo:+.4f}, {r.ci_hi:+.4f}]", axis=1)154 tex_table(plc, TB / "tab08_placebo.tex", [("placebo", "Placebo", "str"), ("draw", "Draw", "int"), ("n", "N", "int"), ("auc_a", "AUC LB5", "{:.3f}"), ("auc_b", "AUC LB2", "{:.3f}"), ("mean", "$\\Delta$ log loss", "{:+.4f}"), ("ci", "95\\% CI", "str")])155 macro("placeboMeanD", plc["mean"].mean(), "{:+.4f}"); macro("placeboAUC", plc["auc_a"].mean(), "{:.3f}")156 # ---------------- Table 11: economic value157 h5 = json.load(open(E / "h5_portfolios.json")); car = pd.read_csv(E / "h5_car_regressions.csv")158 rows = []159 for m, v in h5.items():160 for k in ("gross", "net20bps"):161 a = v[k]; rows.append({"model": m, "variant": k, "alpha": 100 * a["alpha_monthly"], "t": a["alpha_t"], "mean_ls": 100 * a["mean_ls"], "sharpe": a["sharpe_ann"], "T": a["T"], "turnover": v["avg_turnover"]})162 tex_table(pd.DataFrame(rows), TB / "tab11_economic.tex", group="model", cols=[("model", "Score", "str"), ("variant", "Variant", "str"), ("mean_ls", "Mean L/S (\\%/mo)", "{:.2f}"), ("alpha", "FF5+MOM $\\alpha$ (\\%/mo)", "{:+.2f}"),163 ("t", "$t$ (HAC)", "{:.2f}"), ("sharpe", "Sharpe (ann.)", "{:.2f}"), ("turnover", "Turnover", "{:.2f}"), ("T", "Months", "int")])164 for m, v in h5.items():165 macro(f"alpha{m}gross", 100 * v["gross"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}gross", v["gross"]["alpha_t"], "{:.2f}")166 macro(f"alpha{m}net", 100 * v["net20bps"]["alpha_monthly"], "{:+.2f}"); macro(f"alphat{m}net", v["net20bps"]["alpha_t"], "{:.2f}")167 carr = car[car["var"].isin(["lfds5", "lfds2", "d_lfds"])].copy(); carr["coef_s"] = carr.apply(lambda r: f"{r.coef:.4f}{stars(r.p)}", axis=1); carr["se_s"] = carr.se.map(lambda v: f"({v:.4f})")168 tex_table(carr, TB / "tab11b_car.tex", group="spec", cols=[("spec", "Specification", "str"), ("var", "Regressor", "str"), ("coef_s", "Coef.", "str"), ("se_s", "SE", "str"), ("n", "N", "int")])169 for _, r in carr.iterrows():170 macro(f"car{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.coef, "{:+.4f}"); macro(f"cart{r.spec.replace('_', '')}{r['var'].replace('_', '')}", r.t, "{:.2f}")171 # continuous + power172 cont = pd.read_csv(E / "continuous.csv")173 tex_table(cont[cont.model.isin(["B2", "B5"])], TB / "tab10b_continuous.tex", [("model", "Model", "str"), ("n", "N", "int"), ("mae", "MAE", "{:.5f}"), ("rmse", "RMSE", "{:.5f}"), ("oos_r2", "OOS $R^2$", "{:.4f}")])174 for _, r in cont.iterrows():175 if r.model in ("B2", "B5"):176 macro(f"r2{r.model}", r.oos_r2, "{:.4f}")177 pw = json.load(open(E / "power.json")); macro("mde", pw.get("mde_logloss_80pct_power", float("nan")), "{:.4f}"); macro("valQuarters", pw.get("validation_quarters", 0))178 # ---------------- figures179 F.fig_timeline(FG)180 yrs = e20.assign(year=pd.to_datetime(e20.cutoff).dt.year).groupby("year").agg(firms=("entity_id", "nunique"), firm_quarters=("entity_id", "size"), adverse_rate=("y_adverse_opinc", "mean")).reset_index()181 F.fig_coverage(FG, yrs)182 te = e20[pd.to_datetime(e20.cutoff) >= pd.Timestamp(str(cfg["splits"]["test"][0]))].copy()183 prev_rows = [{"label": "All firm-quarters", "sel": np.ones(len(te), bool)}]184 for name, col, lab in (("size", "f_log_mcap", "Market cap"), ("illiq", "wd_amihud60", "Illiquidity"), ("lev", "f_leverage", "Leverage")):185 q = te[col].rank(pct=True)186 prev_rows += [{"label": f"{lab}: bottom tercile", "sel": (q <= 1 / 3).values}, {"label": f"{lab}: top tercile", "sel": (q > 2 / 3).values}]187 for rt in ("before_open", "after_close"):188 prev_rows.append({"label": f"Release {rt.replace('_', ' ')}", "sel": (te.release_time == rt).values})189 pr_ = []190 for r in prev_rows:191 y = te.y_adverse_opinc.values[r["sel"]]; n = len(y); p = y.mean() if n else np.nan; se = np.sqrt(p * (1 - p) / n) if n else np.nan192 pr_.append({"label": r["label"], "rate": p, "lo": p - 1.96 * se, "hi": p + 1.96 * se})193 F.fig_prevalence(FG, pd.DataFrame(pr_))194 F.fig_frequency_gains(FG, con)195 F.fig_reliability(FG, pd.read_csv(E / "reliability.csv"))196 F.fig_signal_evolution(FG, lad)197 F.fig_heterogeneity(FG, sub, ["all", "size_small", "size_large", "illiq_high", "illiq_low", "lev_high", "lev_low", "loss_firms", "profit_firms",198 "covid_2020", "ex_covid", "window_clean", "window_straddles_prev_release", "dense_bars", "sparse_bars", "sector_finance", "sector_nonfinance"])199 F.fig_rolling(FG, pd.read_csv(E / "walkforward_by_year.csv"), pd.read_csv(E / "walkforward_contrasts.csv"))200 if "LB5" in h5:201 F.fig_economic(FG, pd.read_csv(E / "portfolios_LB5.csv"), h5["LB5"])202 # ---------------- macros file203 lines = ["% Auto-generated by scripts/09_make_manuscript.py — do not edit"]204 for k, v in T.items():205 words = {"0": "Zero", "1": "One", "2": "Two", "3": "Three", "4": "Four", "5": "Five", "6": "Six", "7": "Seven", "8": "Eight", "9": "Nine"}206 kk = "".join(words.get(ch, ch) if ch.isdigit() else ch for ch in k if ch.isalnum())207 lines.append(f"\\newcommand{{\\{kk}}}{{{v}}}")208 (P.root / "paper" / "results_macros.tex").write_text("\n".join(lines) + "\n")209 log.info("wrote %d macros, %d tables, figures to %s", len(T), len(list(TB.glob('*.tex'))), FG)210 return 0211212213if __name__ == "__main__":214 raise SystemExit(main())215