Python 56.2%
TeX 43.7%
1"""Clean Polymarket raw data and build structured panels for analysis."""23from __future__ import annotations45import json6from pathlib import Path78import numpy as np9import pandas as pd1011from src.config import (12 DATA_PROCESSED,13 DATA_RAW_POLY,14 ELECTION_2024_EVENT_ID,15 FED_JUNE_2026_EVENT_ID,16)17from src.utils import setup_logging1819logger = setup_logging("polymarket_cleaner")202122# ---------------------------------------------------------------------------23# Low-level loaders24# ---------------------------------------------------------------------------2526def load_event_markets(event_id: int) -> list[dict]:27 """Load event JSON and extract market info with token IDs."""28 path = DATA_RAW_POLY / f"event_{event_id}.json"29 if not path.exists():30 logger.warning("Event file not found: %s", path)31 return []3233 with open(path) as f:34 event_data = json.load(f)3536 results: list[dict] = []37 for m in event_data.get("markets", []):38 tokens = json.loads(m.get("clobTokenIds", "[]"))39 if tokens:40 results.append({41 "id": m["id"],42 "name": m.get("groupItemTitle", m.get("question", "")[:40]),43 "question": m.get("question", ""),44 "yes_token": tokens[0],45 "no_token": tokens[1] if len(tokens) > 1 else "",46 "closed": m.get("closed", False),47 "prices": m.get("outcomePrices", ""),48 "volume": float(m.get("volumeNum", m.get("volume", 0) or 0)),49 })50 logger.info("Loaded %d markets from event %d", len(results), event_id)51 return results525354def load_price_history(token_id: str) -> pd.DataFrame:55 """Load price history JSON for a token, return DataFrame with DatetimeIndex.5657 Files are named ``price_history_{token_id[:16]}.json``.58 Content is a list of ``{"t": unix_ts, "p": float}``.59 Returns a DataFrame with a datetime index (``date``) and a ``price`` column.60 """61 short_id = token_id[:16]62 path = DATA_RAW_POLY / f"price_history_{short_id}.json"63 if not path.exists():64 logger.warning("Price history file not found: %s", path)65 return pd.DataFrame(columns=["price"])6667 with open(path) as f:68 raw = json.load(f)6970 if not raw:71 logger.warning("Empty price history for token %s", short_id)72 return pd.DataFrame(columns=["price"])7374 df = pd.DataFrame(raw)75 df["date"] = pd.to_datetime(df["t"], unit="s", utc=True).dt.normalize()76 df["price"] = df["p"].astype(float)77 # Some data may have duplicates for the same day (intra-day ticks);78 # keep the last observation per calendar day.79 df = df.drop_duplicates(subset="date", keep="last")80 df = df.set_index("date").sort_index()[["price"]]81 df.index = df.index.tz_localize(None) # drop tz for easier merging82 logger.info("Loaded price history for %s: %d observations", short_id, len(df))83 return df848586# ---------------------------------------------------------------------------87# Panel builders88# ---------------------------------------------------------------------------8990def build_market_panel(event_id: int) -> pd.DataFrame:91 """Build panel of prices for all markets in an event.9293 Columns = market names (``groupItemTitle``), Index = date, Values = prices.94 """95 markets = load_event_markets(event_id)96 if not markets:97 logger.warning("No markets found for event %d", event_id)98 return pd.DataFrame()99100 series_dict: dict[str, pd.Series] = {}101 for m in markets:102 df = load_price_history(m["yes_token"])103 if not df.empty:104 series_dict[m["name"]] = df["price"]105106 if not series_dict:107 logger.warning("No price data for event %d", event_id)108 return pd.DataFrame()109110 panel = pd.DataFrame(series_dict)111 panel = panel.sort_index()112 # Forward-fill small gaps (weekends / missing days)113 panel = panel.ffill(limit=3)114 logger.info(115 "Built market panel for event %d: %d dates x %d markets",116 event_id, len(panel), len(panel.columns),117 )118 return panel119120121def build_arrow_debreu_state_prices(event_id: int = FED_JUNE_2026_EVENT_ID) -> pd.DataFrame:122 """Build Arrow-Debreu state-price matrix for the Fed June 2026 neg-risk group.123124 Each market corresponds to one Arrow-Debreu state125 (50+ bps decrease, 25 bps decrease, no change, 25 bps increase, 50+ bps increase).126 The state price equals the market YES price.127128 Adds:129 ``state_price_sum`` -- sum of state prices per day (should be close to 1.0).130 ``completeness_deviation`` -- ``state_price_sum - 1.0``.131 """132 panel = build_market_panel(event_id)133 if panel.empty:134 return panel135136 # Canonical ordering of the state space137 canonical_order = [138 "50+ bps decrease",139 "25 bps decrease",140 "No change",141 "25 bps increase",142 "50+ bps increase",143 ]144 # Reorder columns to canonical order, keeping only those present145 cols = [c for c in canonical_order if c in panel.columns]146 panel = panel[cols]147148 panel["state_price_sum"] = panel[cols].sum(axis=1)149 panel["completeness_deviation"] = panel["state_price_sum"] - 1.0150151 logger.info(152 "Arrow-Debreu state prices: mean sum = %.4f, mean |deviation| = %.4f",153 panel["state_price_sum"].mean(),154 panel["completeness_deviation"].abs().mean(),155 )156 return panel157158159def build_election_panel() -> pd.DataFrame:160 """Build panel for Election 2024 markets (Trump, Biden/Harris, etc.)."""161 return build_market_panel(ELECTION_2024_EVENT_ID)162163164# ---------------------------------------------------------------------------165# Master cleaning166# ---------------------------------------------------------------------------167168def clean_all() -> None:169 """Run all Polymarket cleaning and save Parquet files to ``data/processed/``."""170 DATA_PROCESSED.mkdir(parents=True, exist_ok=True)171172 # 1. Fed June 2026 -- Arrow-Debreu state prices173 fed_panel = build_arrow_debreu_state_prices()174 if not fed_panel.empty:175 out = DATA_PROCESSED / "fed_june2026_state_prices.parquet"176 fed_panel.to_parquet(out)177 logger.info("Saved %s (%d rows)", out.name, len(fed_panel))178179 # 2. Fed June 2026 -- raw market panel (without AD columns)180 fed_raw = build_market_panel(FED_JUNE_2026_EVENT_ID)181 if not fed_raw.empty:182 out = DATA_PROCESSED / "fed_june2026_panel.parquet"183 fed_raw.to_parquet(out)184 logger.info("Saved %s (%d rows)", out.name, len(fed_raw))185186 # 3. Election 2024 panel187 election = build_election_panel()188 if not election.empty:189 out = DATA_PROCESSED / "election_2024_panel.parquet"190 election.to_parquet(out)191 logger.info("Saved %s (%d rows)", out.name, len(election))192193 logger.info("=== Polymarket cleaning complete ===")194