TeX 53.6%
Python 46.2%
1"""Stage A pilot figures and summary-statistics table (CLAUDE.md §33.11-12).23Figure P1 — dataset coverage: monthly compressed size (all 411 months, from the4pinned inventory) and symbols per trading day (three pilot months).5Figure P2 — intraday seasonality by minute of day for the pilot months, computed6on the top-500 dollar-volume universe within each month (descriptive, full-month7normalization — documented; predictive designs will use lagged estimates).8Table P1 — summary statistics of core minute-level variables, same universe.910Each figure's source data is saved to outputs/figures/data/ so the figure can be11reproduced without re-running this script's heavy pass.12"""1314from __future__ import annotations1516import json17import sys18from pathlib import Path1920import matplotlib2122matplotlib.use("Agg")23import matplotlib.pyplot as plt24import numpy as np25import polars as pl2627ROOT = Path(__file__).resolve().parents[1]28sys.path.insert(0, str(ROOT / "src"))2930RAW = ROOT / "data" / "raw" / "data"31FIGDIR = ROOT / "outputs" / "figures"32FIGDATA = FIGDIR / "data"33TABDIR = ROOT / "outputs" / "tables"3435MONTHS = ["1992-06", "2009-06", "2025-12"]36COLORS = {"1992-06": "#0072B2", "2009-06": "#D55E00", "2025-12": "#009E73"}37STYLES = {"1992-06": "-", "2009-06": "--", "2025-12": ":"}38TOP_N = 5003940plt.rcParams.update({41 "font.family": "serif",42 "font.size": 9,43 "axes.spines.top": False,44 "axes.spines.right": False,45 "axes.grid": True,46 "grid.color": "0.9",47 "grid.linewidth": 0.5,48 "axes.axisbelow": True,49 "figure.facecolor": "white",50 "savefig.facecolor": "white",51 "pdf.fonttype": 42,52})535455def load_month_rth(month: str) -> pl.DataFrame:56 """RTH bars with ET clock fields for one pilot month."""57 df = pl.read_parquet(RAW / f"ohlcv_{month}.parquet")58 et = df.with_columns(59 pl.col("timestamp").dt.convert_time_zone("America/New_York").alias("ts_et"))60 et = et.with_columns(61 pl.col("ts_et").dt.date().alias("date"),62 (pl.col("ts_et").dt.hour().cast(pl.Int32) * 6063 + pl.col("ts_et").dt.minute()).alias("mod"),64 )65 return et.filter((pl.col("mod") >= 570) & (pl.col("mod") < 960))666768def top_universe(rth: pl.DataFrame, n: int = TOP_N) -> list[str]:69 dv = (rth.group_by("ticker")70 .agg((pl.col("close") * pl.col("volume")).sum().alias("dollar_vol"))71 .sort("dollar_vol", descending=True))72 return dv.head(n)["ticker"].to_list()737475def seasonality(rth: pl.DataFrame, universe: list[str]) -> pl.DataFrame:76 """Minute-of-day means of |log return| (bps), volume share, log range (bps)."""77 u = rth.filter(pl.col("ticker").is_in(universe)).sort("ticker", "date", "mod")78 u = u.with_columns(79 (pl.col("close").log() - pl.col("close").log().shift(1)80 ).over("ticker", "date").alias("ret"),81 (pl.col("high").log() - pl.col("low").log()).alias("rng"),82 )83 day_vol = u.group_by("ticker", "date").agg(pl.col("volume").sum().alias("dvol"))84 u = u.join(day_vol, on=["ticker", "date"]).with_columns(85 (pl.col("volume") / pl.col("dvol")).alias("vshare"))86 return (u.group_by("mod").agg(87 (pl.col("ret").abs().mean() * 1e4).alias("absret_bps"),88 (pl.col("vshare").mean() * 100).alias("volshare_pct"),89 (pl.col("rng").mean() * 1e4).alias("range_bps"),90 pl.len().alias("n_bars"),91 ).sort("mod"))929394def summary_stats(rth: pl.DataFrame, universe: list[str], month: str) -> pl.DataFrame:95 u = rth.filter(pl.col("ticker").is_in(universe)).sort("ticker", "date", "mod")96 u = u.with_columns(97 ((pl.col("close").log() - pl.col("close").log().shift(1)) * 1e498 ).over("ticker", "date").alias("ret_bps"),99 ((pl.col("high").log() - pl.col("low").log()) * 1e4).alias("range_bps"),100 (pl.col("close") * pl.col("volume") / 1e3).alias("dollar_vol_k"),101 )102 bars = u.group_by("ticker", "date").agg(pl.len().alias("bars_per_day"))103 rows = []104 for var, series in [105 ("Minute log return (bps)", u["ret_bps"].drop_nulls()),106 ("High--low log range (bps)", u["range_bps"]),107 ("Dollar volume per bar (\\$000)", u["dollar_vol_k"]),108 ("Valid RTH bars per stock-day", bars["bars_per_day"].cast(pl.Float64)),109 ]:110 rows.append({111 "month": month, "variable": var,112 "mean": float(series.mean()), "sd": float(series.std()),113 "p1": float(series.quantile(0.01)), "p5": float(series.quantile(0.05)),114 "p50": float(series.median()),115 "p95": float(series.quantile(0.95)), "p99": float(series.quantile(0.99)),116 "n": series.len(),117 })118 return pl.DataFrame(rows)119120121def fig_coverage() -> None:122 inv = json.loads((ROOT / "data/manifests/repo_inventory.json").read_text())123 rec = [(f["path"][-15:-8], f["size"] / 1e6) for f in inv["files"]124 if f["path"].endswith(".parquet")]125 rec.sort()126 x = [int(m[:4]) + (int(m[5:7]) - 0.5) / 12 for m, _ in rec]127 y = [s for _, s in rec]128 pl.DataFrame({"month": [m for m, _ in rec], "mb_compressed": y}).write_csv(129 FIGDATA / "figP1a_monthly_file_size.csv")130131 per_day = {}132 for month in MONTHS:133 rep = json.loads(134 (ROOT / f"outputs/diagnostics/pilot_audit_{month}.json").read_text())135 per_day[month] = [(d["date"], d["symbols"]) for d in rep["per_day"]]136 pl.DataFrame([137 {"month": m, "date": d, "symbols": s}138 for m, days in per_day.items() for d, s in days139 ]).write_csv(FIGDATA / "figP1b_symbols_per_day.csv")140141 fig, axes = plt.subplots(1, 2, figsize=(7.0, 2.6))142 ax = axes[0]143 ax.plot(x, y, color="#0072B2", lw=1.2)144 ax.set_xlabel("Year")145 ax.set_ylabel("Compressed file size (MB / month)")146 ax.set_title("(a) Raw data volume by month", fontsize=9, loc="left")147 ax.set_xlim(1992, 2026.5)148149 ax = axes[1]150 for month in MONTHS:151 days = list(range(1, len(per_day[month]) + 1))152 sym = [s for _, s in per_day[month]]153 ax.plot(days, sym, STYLES[month], color=COLORS[month], lw=1.4)154 ax.annotate(month, xy=(days[-1], sym[-1]), xytext=(3, 0),155 textcoords="offset points", color=COLORS[month],156 fontsize=8, va="center")157 ax.set_xlabel("Trading day of month")158 ax.set_ylabel("Symbols with $\\geq$1 bar")159 ax.set_title("(b) Active symbols per day, pilot months", fontsize=9, loc="left")160 ax.set_xlim(1, 27)161162 fig.tight_layout()163 for ext in ("pdf", "png"):164 fig.savefig(FIGDIR / f"figP1_coverage.{ext}", dpi=300)165 plt.close(fig)166167168def fig_seasonality(seas: dict[str, pl.DataFrame]) -> None:169 panels = [170 ("absret_bps", "(a) $|$log return$|$ (bps)"),171 ("volshare_pct", "(b) Volume share (%)"),172 ("range_bps", "(c) High–low range (bps)"),173 ]174 fig, axes = plt.subplots(1, 3, figsize=(7.0, 2.4), sharex=True)175 for ax, (col, title) in zip(axes, panels):176 for month in MONTHS:177 s = seas[month]178 hours = (s["mod"] - 570) / 60.0179 ax.plot(hours, s[col], STYLES[month], color=COLORS[month], lw=1.2,180 label=month)181 ax.set_title(title, fontsize=9, loc="left")182 ax.set_xlabel("Hours since 09:30 ET")183 ax.set_xlim(0, 6.5)184 ax.set_xticks([0, 2, 4, 6])185 axes[0].legend(frameon=False, fontsize=7.5, loc="upper right",186 borderaxespad=0.1, handlelength=1.8)187 fig.tight_layout()188 for ext in ("pdf", "png"):189 fig.savefig(FIGDIR / f"figP2_seasonality.{ext}", dpi=300)190 plt.close(fig)191192193def latex_table(stats: pl.DataFrame) -> str:194 month_label = {"1992-06": "June 1992", "2009-06": "June 2009",195 "2025-12": "December 2025"}196 lines = [197 "% Auto-generated by scripts/02_pilot_figures_tables.py — do not edit numbers by hand.",198 "\\begin{table}[t]",199 "\\centering",200 "\\begin{adjustbox}{max width=\\textwidth}\\begin{threeparttable}",201 "\\caption{Summary statistics, pilot months}",202 "\\label{tab:pilot_summary}",203 "\\small",204 "\\begin{tabular}{llrrrrrrrr}",205 "\\toprule",206 "Month & Variable & Mean & SD & p1 & p5 & p50 & p95 & p99 & $N$ \\\\",207 "\\midrule",208 ]209 fmt = lambda v: f"{v:,.2f}" if abs(v) < 1e5 else f"{v:,.0f}"210 prev = None211 for r in stats.iter_rows(named=True):212 m = month_label[r["month"]] if r["month"] != prev else ""213 if r["month"] != prev and prev is not None:214 lines.append("\\addlinespace")215 prev = r["month"]216 lines.append(217 f"{m} & {r['variable']} & {fmt(r['mean'])} & {fmt(r['sd'])} & "218 f"{fmt(r['p1'])} & {fmt(r['p5'])} & {fmt(r['p50'])} & "219 f"{fmt(r['p95'])} & {fmt(r['p99'])} & {r['n']:,} \\\\")220 lines += [221 "\\bottomrule",222 "\\end{tabular}",223 "\\begin{tablenotes}[flushleft]\\footnotesize",224 "\\item Notes: Regular trading hours (09:30--16:00 ET, exchange calendar) for the "225 f"top {TOP_N} securities by within-month dollar volume in each pilot month of the "226 "pinned Finnhub.io minute-bar aggregation. Minute log "227 "returns use consecutive same-day closes; the high--low range is "228 "$\\log(H)-\\log(L)$ per one-minute bar; bars exist only for minutes with at "229 "least one reported trade. Prices are unadjusted for splits; the pilot is "230 "descriptive and imposes no corporate-action screens.",231 "\\end{tablenotes}",232 "\\end{threeparttable}\\end{adjustbox}",233 "\\end{table}",234 ]235 return "\n".join(lines)236237238def main() -> None:239 for d in (FIGDATA, TABDIR):240 d.mkdir(parents=True, exist_ok=True)241242 seas, stats_frames = {}, []243 for month in MONTHS:244 rth = load_month_rth(month)245 uni = top_universe(rth)246 seas[month] = seasonality(rth, uni)247 seas[month].write_csv(FIGDATA / f"figP2_seasonality_{month}.csv")248 stats_frames.append(summary_stats(rth, uni, month))249 print(f"{month}: universe={len(uni)} seasonality rows={seas[month].height}")250251 stats = pl.concat(stats_frames)252 stats.write_csv(TABDIR / "tabP1_pilot_summary.csv")253 tex = latex_table(stats)254 (TABDIR / "tabP1_pilot_summary.tex").write_text(tex)255 (ROOT / "paper" / "tables" / "tabP1_pilot_summary.tex").write_text(tex)256257 fig_coverage()258 fig_seasonality(seas)259 for f in ("figP1_coverage.pdf", "figP2_seasonality.pdf"):260 (ROOT / "paper" / "figures" / f).write_bytes((FIGDIR / f).read_bytes())261 print("figures + table written")262263264if __name__ == "__main__":265 main()266