Python 56.2%
TeX 43.7%
1"""Master analysis pipeline for the Polymarket Arrow-Debreu paper.23Usage: python scripts/run_analysis.py [--skip-data] [--skip-models] [--skip-paper]4"""56from __future__ import annotations78import argparse9import json10import sys11from pathlib import Path1213sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # make `src` importable1415import numpy as np16import pandas as pd1718from src.config import (19 DATA_PROCESSED,20 DATA_RAW_POLY,21 ELECTION_KEY_DATES,22 FOMC_DATES,23 CPI_RELEASE_DATES,24)25from src.utils import ensure_dirs, setup_logging2627logger = setup_logging("analysis")282930# ── Phase 1: Data acquisition ────────────────────────────────────────────────3132def run_data_acquisition() -> None:33 import asyncio34 from src.api.runner import run_all35 run_all()363738# ── Phase 2: Cleaning ────────────────────────────────────────────────────────3940def run_cleaning() -> None:41 from src.cleaning.polymarket_cleaner import clean_all as clean_poly42 from src.cleaning.macro_cleaner import clean_all as clean_macro43 from src.cleaning.yahoo_cleaner import clean_all as clean_yahoo44 clean_poly()45 clean_macro()46 clean_yahoo()474849# ── Phase 3: Models + Figures + Tables ────────────────────────────────────────5051def run_models_and_outputs() -> None:52 from src.models.arrow_debreu import (53 test_market_completeness,54 extract_risk_neutral_density,55 compute_risk_neutral_moments,56 detect_arbitrage_violations,57 state_price_dynamics,58 )59 from src.models.event_study import run_event_study60 from src.models.price_discovery import granger_causality_matrix, estimate_var61 from src.models.volatility import (62 compute_realized_volatility_measures,63 estimate_har_rv,64 estimate_garch,65 )66 from src.models.liquidity import analyze_volume_patterns, market_quality_from_orderbook67 from src.features.engineering import compute_roll_spread68 from src.plots.figures import (69 fig_state_prices,70 fig_completeness_test,71 fig_risk_neutral_density,72 fig_event_study_cars,73 fig_granger_heatmap,74 fig_impulse_response,75 fig_volatility,76 fig_liquidity,77 fig_election_case_study,78 fig_arbitrage_violations,79 )80 from src.plots.tables import (81 table_descriptive_stats,82 table_summary_stats,83 table_completeness_test,84 table_event_study,85 table_granger_causality,86 table_var_results,87 table_garch,88 table_har_rv,89 table_liquidity,90 )91 from src.plots.style import save_figure9293 # ── Load cleaned data ─────────────────────────────────────────────────94 logger.info("Loading cleaned data...")95 fed_state = pd.read_parquet(DATA_PROCESSED / "fed_june2026_state_prices.parquet")96 fed_panel = pd.read_parquet(DATA_PROCESSED / "fed_june2026_panel.parquet")97 election = pd.read_parquet(DATA_PROCESSED / "election_2024_panel.parquet")98 financial = pd.read_parquet(DATA_PROCESSED / "financial_data.parquet")99 macro_events = pd.read_parquet(DATA_PROCESSED / "macro_events.parquet")100101 state_cols = [c for c in fed_state.columns if c not in ("state_price_sum", "completeness_deviation")]102 state_prices = fed_state[state_cols]103104 # ── 1. Arrow-Debreu completeness tests ────────────────────────────────105 logger.info("=== Arrow-Debreu Completeness Tests ===")106 completeness = test_market_completeness(state_prices)107 logger.info(" Mean sum: %.4f, Std: %.4f, All positive: %s",108 completeness.mean_sum, completeness.std_sum, completeness.all_positive)109110 state_values = [-50.0, -25.0, 0.0, 25.0, 50.0]111 rn_density = extract_risk_neutral_density(state_prices, state_values)112 rn_moments = compute_risk_neutral_moments(rn_density, state_values)113 arb_violations = detect_arbitrage_violations(state_prices, market_type="state_space")114 dynamics = state_price_dynamics(state_prices)115116 # Figures117 fig = fig_state_prices(state_prices)118 save_figure(fig, "fig1_state_prices")119 logger.info(" Saved fig1_state_prices")120121 fig = fig_completeness_test(completeness.daily_sums)122 save_figure(fig, "fig2_completeness_test")123 logger.info(" Saved fig2_completeness_test")124125 fig = fig_risk_neutral_density(rn_density, state_values)126 save_figure(fig, "fig3_risk_neutral_density")127 logger.info(" Saved fig3_risk_neutral_density")128129 fig = fig_arbitrage_violations(arb_violations)130 save_figure(fig, "fig12_arbitrage_violations")131 logger.info(" Saved fig12_arbitrage_violations")132133 # Tables134 table_completeness_test(completeness)135 logger.info(" Saved table3_completeness")136 table_summary_stats(state_prices)137 logger.info(" Saved table2_summary_stats")138139 # ── 2. Event study ────────────────────────────────────────────────────140 logger.info("=== Event Study ===")141 fomc_dates_in_range = [d for d in FOMC_DATES if fed_state.index.min().date() <= d <= fed_state.index.max().date()]142 cpi_dates_in_range = [d for d in CPI_RELEASE_DATES if fed_state.index.min().date() <= d <= fed_state.index.max().date()]143144 event_results = {}145 if fomc_dates_in_range:146 es_fomc = run_event_study(state_prices, fomc_dates_in_range, "FOMC", window=(-3, 3))147 event_results["FOMC"] = es_fomc148 logger.info(" FOMC: %d events", es_fomc.n_events)149150 if cpi_dates_in_range:151 es_cpi = run_event_study(state_prices, cpi_dates_in_range, "CPI", window=(-3, 3))152 event_results["CPI"] = es_cpi153 logger.info(" CPI: %d events", es_cpi.n_events)154155 # Election event study with broader window156 election_dates = list(ELECTION_KEY_DATES.values())157 election_dates_in_range = [d for d in election_dates if election.index.min().date() <= d <= election.index.max().date()]158 if election_dates_in_range and "Donald Trump" in election.columns:159 es_election = run_event_study(160 election[["Donald Trump"]],161 election_dates_in_range,162 "Election",163 window=(-5, 5),164 )165 event_results["Election"] = es_election166 logger.info(" Election: %d events", es_election.n_events)167168 if event_results:169 cars_for_plot = {}170 for etype, result in event_results.items():171 if not result.test_stats.empty:172 cars_for_plot[etype] = result.test_stats173 if cars_for_plot:174 fig = fig_event_study_cars(cars_for_plot, window=(-3, 3))175 save_figure(fig, "fig5_event_study_cars")176 logger.info(" Saved fig5_event_study_cars")177178 table_event_study(event_results)179 logger.info(" Saved table4_event_study")180181 # ── 3. Price discovery ────────────────────────────────────────────────182 logger.info("=== Price Discovery ===")183 no_change = state_prices.get("No change")184 if no_change is not None and "SPY_log_return" in financial.columns:185 aligned = pd.DataFrame({186 "Fed_NoChange": no_change.pct_change(),187 "SPY": financial["SPY_log_return"],188 "BTC": financial.get("BTC_USD_log_return", pd.Series(dtype=float)),189 "VIX": financial.get("VIX_log_return", pd.Series(dtype=float)),190 }).dropna()191192 if len(aligned) > 20:193 f_stats, p_values = granger_causality_matrix(aligned, max_lag=5)194 fig = fig_granger_heatmap(p_values)195 save_figure(fig, "fig7_granger_heatmap")196 logger.info(" Saved fig7_granger_heatmap")197 table_granger_causality(f_stats, p_values)198 logger.info(" Saved table5_granger")199200 var_result = estimate_var(aligned, max_lags=5)201 if var_result["model"] is not None:202 table_var_results(var_result)203 logger.info(" Saved table6_var")204 if var_result["irf"]:205 fig = fig_impulse_response(var_result["irf"])206 save_figure(fig, "fig8_impulse_response")207 logger.info(" Saved fig8_impulse_response")208 else:209 logger.warning(" Not enough aligned data for price discovery (%d rows)", len(aligned))210 else:211 logger.warning(" Missing data for price discovery analysis")212213 # ── 4. Volatility ─────────────────────────────────────────────────────214 logger.info("=== Volatility Analysis ===")215 if no_change is not None:216 returns = no_change.pct_change().dropna()217 if len(returns) > 30:218 rv_measures = compute_realized_volatility_measures(returns)219220 # HAR-RV221 try:222 har_result = estimate_har_rv(rv_measures)223 table_har_rv(har_result)224 logger.info(" HAR-RV R²: %.4f", har_result.r_squared)225 except Exception as e:226 logger.warning(" HAR-RV failed: %s", e)227 har_result = None228229 # GARCH230 garch_results = {}231 for mtype in ["GARCH", "GJR-GARCH"]:232 try:233 garch_results[mtype] = estimate_garch(returns, model_type=mtype)234 logger.info(" %s AIC: %.2f", mtype, garch_results[mtype].aic)235 except Exception as e:236 logger.warning(" %s failed: %s", mtype, e)237238 if garch_results:239 table_garch(garch_results)240 logger.info(" Saved table7_garch")241242 first_garch = list(garch_results.values())[0]243 fig = fig_volatility(244 rv_measures["rv_5"],245 cond_vol=first_garch.conditional_volatility,246 )247 save_figure(fig, "fig9_volatility")248 logger.info(" Saved fig9_volatility")249250 # ── 5. Liquidity ──────────────────────────────────────────────────────251 logger.info("=== Liquidity Analysis ===")252 # Use volume from discovered markets metadata253 discovered_path = DATA_RAW_POLY / "discovered_markets.json"254 if discovered_path.exists():255 with open(discovered_path) as f:256 discovered = json.load(f)257 volumes = pd.DataFrame([{258 "name": m.get("question", "")[:40],259 "volume": float(m.get("volumeNum", 0)),260 "liquidity": float(m.get("liquidityNum", 0)),261 } for m in discovered if float(m.get("volumeNum", 0)) > 0])262263 if not volumes.empty:264 metadata = pd.DataFrame([{265 "name": m.get("question", "")[:40],266 "volume": float(m.get("volumeNum", 0)),267 "liquidity": float(m.get("liquidityNum", 0)),268 "n_obs": 0,269 "mean_price": np.nan,270 "std_price": np.nan,271 } for m in discovered[:20]])272 table_descriptive_stats(metadata)273 logger.info(" Saved table1_descriptive_stats")274275 # Roll spread for Fed panel276 if no_change is not None:277 roll_spread = compute_roll_spread(no_change)278279 fig = fig_liquidity(280 no_change.rename("Price"),281 spreads=roll_spread,282 )283 save_figure(fig, "fig10_liquidity")284 logger.info(" Saved fig10_liquidity")285286 # ── 6. Election case study ────────────────────────────────────────────287 logger.info("=== Election Case Study ===")288 if "Donald Trump" in election.columns:289 trump = election["Donald Trump"]290 spy_ret = financial.get("SPY_log_return")291 fig = fig_election_case_study(trump, spy_returns=spy_ret)292 save_figure(fig, "fig11_election_case_study")293 logger.info(" Saved fig11_election_case_study")294295 logger.info("=== All analysis complete ===")296297298# ── Main ──────────────────────────────────────────────────────────────────────299300def main() -> None:301 parser = argparse.ArgumentParser(description="Polymarket Arrow-Debreu Analysis Pipeline")302 parser.add_argument("--skip-data", action="store_true", help="Skip data acquisition")303 parser.add_argument("--skip-models", action="store_true", help="Skip models/figures/tables")304 parser.add_argument("--skip-paper", action="store_true", help="Skip paper compilation")305 args = parser.parse_args()306307 ensure_dirs()308309 if not args.skip_data:310 logger.info("Phase 1: Data acquisition")311 run_data_acquisition()312 logger.info("Phase 2: Cleaning")313 run_cleaning()314315 if not args.skip_models:316 logger.info("Phase 3-4: Models, figures, and tables")317 run_models_and_outputs()318319 if not args.skip_paper:320 logger.info("Phase 5: Paper compilation")321 import subprocess322 result = subprocess.run(323 ["latexmk", "-pdf", "-interaction=nonstopmode", "main.tex"],324 cwd="paper",325 capture_output=True,326 text=True,327 )328 if result.returncode == 0:329 logger.info("Paper compiled successfully")330 else:331 logger.warning("Paper compilation had issues (check paper/main.log)")332333334if __name__ == "__main__":335 main()336