Python 56.2%
TeX 43.7%
1"""Master analysis pipeline for the Polymarket Arrow-Debreu paper (v1.0 pipeline).23Usage: python scripts/run_analysis.py [--skip-data] [--skip-models] [--skip-paper]45Revision 2026-10-17: phases 1-2 (download, cleaning) are still used; the tables and6figures of the revised paper (v1.1) are produced by ``scripts/02_revision_analysis.py``,7which supersedes phase 3 of this script. Phase 3 is kept for reproducibility of v1.0.8"""910from __future__ import annotations1112import argparse13import json14import sys15from pathlib import Path1617sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # make `src` importable1819import numpy as np20import pandas as pd2122from src.config import (23 DATA_PROCESSED,24 DATA_RAW_POLY,25 ELECTION_KEY_DATES,26 FOMC_DATES,27 CPI_RELEASE_DATES,28)29from src.utils import ensure_dirs, setup_logging3031logger = setup_logging("analysis")323334# ── Phase 1: Data acquisition ────────────────────────────────────────────────3536def run_data_acquisition() -> None:37 import asyncio38 from src.api.runner import run_all39 run_all()404142# ── Phase 2: Cleaning ────────────────────────────────────────────────────────4344def run_cleaning() -> None:45 from src.cleaning.polymarket_cleaner import clean_all as clean_poly46 from src.cleaning.macro_cleaner import clean_all as clean_macro47 from src.cleaning.yahoo_cleaner import clean_all as clean_yahoo48 clean_poly()49 clean_macro()50 clean_yahoo()515253# ── Phase 3: Models + Figures + Tables ────────────────────────────────────────5455def run_models_and_outputs() -> None:56 from src.models.arrow_debreu import (57 test_market_completeness,58 extract_risk_neutral_density,59 compute_risk_neutral_moments,60 detect_arbitrage_violations,61 state_price_dynamics,62 )63 from src.models.event_study import run_event_study64 from src.models.price_discovery import granger_causality_matrix, estimate_var65 from src.models.volatility import (66 compute_realized_volatility_measures,67 estimate_har_rv,68 estimate_garch,69 )70 from src.models.liquidity import analyze_volume_patterns, market_quality_from_orderbook71 from src.features.engineering import compute_roll_spread72 from src.plots.figures import (73 fig_state_prices,74 fig_completeness_test,75 fig_risk_neutral_density,76 fig_event_study_cars,77 fig_granger_heatmap,78 fig_impulse_response,79 fig_volatility,80 fig_liquidity,81 fig_election_case_study,82 fig_arbitrage_violations,83 )84 from src.plots.tables import (85 table_descriptive_stats,86 table_summary_stats,87 table_completeness_test,88 table_event_study,89 table_granger_causality,90 table_var_results,91 table_garch,92 table_har_rv,93 table_liquidity,94 )95 from src.plots.style import save_figure9697 # ── Load cleaned data ─────────────────────────────────────────────────98 logger.info("Loading cleaned data...")99 fed_state = pd.read_parquet(DATA_PROCESSED / "fed_june2026_state_prices.parquet")100 fed_panel = pd.read_parquet(DATA_PROCESSED / "fed_june2026_panel.parquet")101 election = pd.read_parquet(DATA_PROCESSED / "election_2024_panel.parquet")102 financial = pd.read_parquet(DATA_PROCESSED / "financial_data.parquet")103 macro_events = pd.read_parquet(DATA_PROCESSED / "macro_events.parquet")104105 state_cols = [c for c in fed_state.columns if c not in ("state_price_sum", "completeness_deviation")]106 state_prices = fed_state[state_cols]107108 # ── 1. Arrow-Debreu completeness tests ────────────────────────────────109 logger.info("=== Arrow-Debreu Completeness Tests ===")110 completeness = test_market_completeness(state_prices)111 logger.info(" Mean sum: %.4f, Std: %.4f, All positive: %s",112 completeness.mean_sum, completeness.std_sum, completeness.all_positive)113114 state_values = [-50.0, -25.0, 0.0, 25.0, 50.0]115 rn_density = extract_risk_neutral_density(state_prices, state_values)116 rn_moments = compute_risk_neutral_moments(rn_density, state_values)117 arb_violations = detect_arbitrage_violations(state_prices, market_type="state_space")118 dynamics = state_price_dynamics(state_prices)119120 # Figures121 fig = fig_state_prices(state_prices)122 save_figure(fig, "fig1_state_prices")123 logger.info(" Saved fig1_state_prices")124125 fig = fig_completeness_test(completeness.daily_sums)126 save_figure(fig, "fig2_completeness_test")127 logger.info(" Saved fig2_completeness_test")128129 fig = fig_risk_neutral_density(rn_density, state_values)130 save_figure(fig, "fig3_risk_neutral_density")131 logger.info(" Saved fig3_risk_neutral_density")132133 fig = fig_arbitrage_violations(arb_violations)134 save_figure(fig, "fig12_arbitrage_violations")135 logger.info(" Saved fig12_arbitrage_violations")136137 # Tables138 table_completeness_test(completeness)139 logger.info(" Saved table3_completeness")140 table_summary_stats(state_prices)141 logger.info(" Saved table2_summary_stats")142143 # ── 2. Event study ────────────────────────────────────────────────────144 logger.info("=== Event Study ===")145 fomc_dates_in_range = [d for d in FOMC_DATES if fed_state.index.min().date() <= d <= fed_state.index.max().date()]146 cpi_dates_in_range = [d for d in CPI_RELEASE_DATES if fed_state.index.min().date() <= d <= fed_state.index.max().date()]147148 event_results = {}149 if fomc_dates_in_range:150 es_fomc = run_event_study(state_prices, fomc_dates_in_range, "FOMC", window=(-3, 3))151 event_results["FOMC"] = es_fomc152 logger.info(" FOMC: %d events", es_fomc.n_events)153154 if cpi_dates_in_range:155 es_cpi = run_event_study(state_prices, cpi_dates_in_range, "CPI", window=(-3, 3))156 event_results["CPI"] = es_cpi157 logger.info(" CPI: %d events", es_cpi.n_events)158159 # Election event study with broader window160 election_dates = list(ELECTION_KEY_DATES.values())161 election_dates_in_range = [d for d in election_dates if election.index.min().date() <= d <= election.index.max().date()]162 if election_dates_in_range and "Donald Trump" in election.columns:163 es_election = run_event_study(164 election[["Donald Trump"]],165 election_dates_in_range,166 "Election",167 window=(-5, 5),168 )169 event_results["Election"] = es_election170 logger.info(" Election: %d events", es_election.n_events)171172 if event_results:173 cars_for_plot = {}174 for etype, result in event_results.items():175 if not result.test_stats.empty:176 cars_for_plot[etype] = result.test_stats177 if cars_for_plot:178 fig = fig_event_study_cars(cars_for_plot, window=(-3, 3))179 save_figure(fig, "fig5_event_study_cars")180 logger.info(" Saved fig5_event_study_cars")181182 table_event_study(event_results)183 logger.info(" Saved table4_event_study")184185 # ── 3. Price discovery ────────────────────────────────────────────────186 logger.info("=== Price Discovery ===")187 no_change = state_prices.get("No change")188 if no_change is not None and "SPY_log_return" in financial.columns:189 aligned = pd.DataFrame({190 "Fed_NoChange": no_change.pct_change(),191 "SPY": financial["SPY_log_return"],192 "BTC": financial.get("BTC_USD_log_return", pd.Series(dtype=float)),193 "VIX": financial.get("VIX_log_return", pd.Series(dtype=float)),194 }).dropna()195196 if len(aligned) > 20:197 f_stats, p_values = granger_causality_matrix(aligned, max_lag=5)198 fig = fig_granger_heatmap(p_values)199 save_figure(fig, "fig7_granger_heatmap")200 logger.info(" Saved fig7_granger_heatmap")201 table_granger_causality(f_stats, p_values)202 logger.info(" Saved table5_granger")203204 var_result = estimate_var(aligned, max_lags=5)205 if var_result["model"] is not None:206 table_var_results(var_result)207 logger.info(" Saved table6_var")208 if var_result["irf"]:209 fig = fig_impulse_response(var_result["irf"])210 save_figure(fig, "fig8_impulse_response")211 logger.info(" Saved fig8_impulse_response")212 else:213 logger.warning(" Not enough aligned data for price discovery (%d rows)", len(aligned))214 else:215 logger.warning(" Missing data for price discovery analysis")216217 # ── 4. Volatility ─────────────────────────────────────────────────────218 logger.info("=== Volatility Analysis ===")219 if no_change is not None:220 returns = no_change.pct_change().dropna()221 if len(returns) > 30:222 rv_measures = compute_realized_volatility_measures(returns)223224 # HAR-RV225 try:226 har_result = estimate_har_rv(rv_measures)227 table_har_rv(har_result)228 logger.info(" HAR-RV R²: %.4f", har_result.r_squared)229 except Exception as e:230 logger.warning(" HAR-RV failed: %s", e)231 har_result = None232233 # GARCH234 garch_results = {}235 for mtype in ["GARCH", "GJR-GARCH"]:236 try:237 garch_results[mtype] = estimate_garch(returns, model_type=mtype)238 logger.info(" %s AIC: %.2f", mtype, garch_results[mtype].aic)239 except Exception as e:240 logger.warning(" %s failed: %s", mtype, e)241242 if garch_results:243 table_garch(garch_results)244 logger.info(" Saved table7_garch")245246 first_garch = list(garch_results.values())[0]247 fig = fig_volatility(248 rv_measures["rv_5"],249 cond_vol=first_garch.conditional_volatility,250 )251 save_figure(fig, "fig9_volatility")252 logger.info(" Saved fig9_volatility")253254 # ── 5. Liquidity ──────────────────────────────────────────────────────255 logger.info("=== Liquidity Analysis ===")256 # Use volume from discovered markets metadata257 discovered_path = DATA_RAW_POLY / "discovered_markets.json"258 if discovered_path.exists():259 with open(discovered_path) as f:260 discovered = json.load(f)261 volumes = pd.DataFrame([{262 "name": m.get("question", "")[:40],263 "volume": float(m.get("volumeNum", 0)),264 "liquidity": float(m.get("liquidityNum", 0)),265 } for m in discovered if float(m.get("volumeNum", 0)) > 0])266267 if not volumes.empty:268 metadata = pd.DataFrame([{269 "name": m.get("question", "")[:40],270 "volume": float(m.get("volumeNum", 0)),271 "liquidity": float(m.get("liquidityNum", 0)),272 "n_obs": 0,273 "mean_price": np.nan,274 "std_price": np.nan,275 } for m in discovered[:20]])276 table_descriptive_stats(metadata)277 logger.info(" Saved table1_descriptive_stats")278279 # Roll spread for Fed panel280 if no_change is not None:281 roll_spread = compute_roll_spread(no_change)282283 fig = fig_liquidity(284 no_change.rename("Price"),285 spreads=roll_spread,286 )287 save_figure(fig, "fig10_liquidity")288 logger.info(" Saved fig10_liquidity")289290 # ── 6. Election case study ────────────────────────────────────────────291 logger.info("=== Election Case Study ===")292 if "Donald Trump" in election.columns:293 trump = election["Donald Trump"]294 spy_ret = financial.get("SPY_log_return")295 fig = fig_election_case_study(trump, spy_returns=spy_ret)296 save_figure(fig, "fig11_election_case_study")297 logger.info(" Saved fig11_election_case_study")298299 logger.info("=== All analysis complete ===")300301302# ── Main ──────────────────────────────────────────────────────────────────────303304def main() -> None:305 parser = argparse.ArgumentParser(description="Polymarket Arrow-Debreu Analysis Pipeline")306 parser.add_argument("--skip-data", action="store_true", help="Skip data acquisition")307 parser.add_argument("--skip-models", action="store_true", help="Skip models/figures/tables")308 parser.add_argument("--skip-paper", action="store_true", help="Skip paper compilation")309 args = parser.parse_args()310311 ensure_dirs()312313 if not args.skip_data:314 logger.info("Phase 1: Data acquisition")315 run_data_acquisition()316 logger.info("Phase 2: Cleaning")317 run_cleaning()318319 if not args.skip_models:320 logger.info("Phase 3-4: Models, figures, and tables")321 run_models_and_outputs()322323 if not args.skip_paper:324 logger.info("Phase 5: Paper compilation")325 import subprocess326 result = subprocess.run(327 ["latexmk", "-pdf", "-interaction=nonstopmode", "main.tex"],328 cwd="paper",329 capture_output=True,330 text=True,331 )332 if result.returncode == 0:333 logger.info("Paper compiled successfully")334 else:335 logger.warning("Paper compilation had issues (check paper/main.log)")336337338if __name__ == "__main__":339 main()340