Python 56.2%
TeX 43.7%
1"""Clean Polymarket raw data and build structured panels for analysis."""23from __future__ import annotations45import json6from pathlib import Path78import numpy as np9import pandas as pd1011from src.config import (12 DATA_PROCESSED,13 DATA_RAW_POLY,14 ELECTION_2024_EVENT_ID,15 FED_JUNE_2026_EVENT_ID,16)17from src.utils import setup_logging1819logger = setup_logging("polymarket_cleaner")202122# ---------------------------------------------------------------------------23# Low-level loaders24# ---------------------------------------------------------------------------2526def load_event_markets(event_id: int) -> list[dict]:27 """Load event JSON and extract market info with token IDs."""28 path = DATA_RAW_POLY / f"event_{event_id}.json"29 if not path.exists():30 logger.warning("Event file not found: %s", path)31 return []3233 with open(path) as f:34 event_data = json.load(f)3536 results: list[dict] = []37 for m in event_data.get("markets", []):38 tokens = json.loads(m.get("clobTokenIds", "[]"))39 if tokens:40 results.append({41 "id": m["id"],42 "name": m.get("groupItemTitle", m.get("question", "")[:40]),43 "question": m.get("question", ""),44 "yes_token": tokens[0],45 "no_token": tokens[1] if len(tokens) > 1 else "",46 "closed": m.get("closed", False),47 "prices": m.get("outcomePrices", ""),48 "volume": float(m.get("volumeNum", m.get("volume", 0) or 0)),49 })50 logger.info("Loaded %d markets from event %d", len(results), event_id)51 return results525354def load_price_history(token_id: str) -> pd.DataFrame:55 """Load price history JSON for a token, return DataFrame with DatetimeIndex.5657 Files are named ``price_history_{token_id[:16]}.json``.58 Content is a list of ``{"t": unix_ts, "p": float}``.59 Returns a DataFrame with a datetime index (``date``) and a ``price`` column.60 """61 short_id = token_id[:16]62 path = DATA_RAW_POLY / f"price_history_{short_id}.json"63 if not path.exists():64 logger.warning("Price history file not found: %s", path)65 return pd.DataFrame(columns=["price"])6667 with open(path) as f:68 raw = json.load(f)6970 if not raw:71 logger.warning("Empty price history for token %s", short_id)72 return pd.DataFrame(columns=["price"])7374 df = pd.DataFrame(raw)75 # The CLOB daily point is stamped at 00:00 UTC of the *following* day and76 # represents the close of the preceding UTC day; label it by the day it77 # closes (revision 2026-10-17; the earlier code labelled it one day late).78 df["date"] = (pd.to_datetime(df["t"], unit="s", utc=True) - pd.Timedelta(minutes=5)).dt.normalize()79 df["price"] = df["p"].astype(float)80 # Some data may have duplicates for the same day (intra-day ticks);81 # keep the last observation per calendar day.82 df = df.drop_duplicates(subset="date", keep="last")83 df = df.set_index("date").sort_index()[["price"]]84 df.index = df.index.tz_localize(None) # drop tz for easier merging85 logger.info("Loaded price history for %s: %d observations", short_id, len(df))86 return df878889# ---------------------------------------------------------------------------90# Panel builders91# ---------------------------------------------------------------------------9293def build_market_panel(event_id: int) -> pd.DataFrame:94 """Build panel of prices for all markets in an event.9596 Columns = market names (``groupItemTitle``), Index = date, Values = prices.97 """98 markets = load_event_markets(event_id)99 if not markets:100 logger.warning("No markets found for event %d", event_id)101 return pd.DataFrame()102103 series_dict: dict[str, pd.Series] = {}104 for m in markets:105 df = load_price_history(m["yes_token"])106 if not df.empty:107 series_dict[m["name"]] = df["price"]108109 if not series_dict:110 logger.warning("No price data for event %d", event_id)111 return pd.DataFrame()112113 panel = pd.DataFrame(series_dict)114 panel = panel.sort_index()115 # Forward-fill small gaps (weekends / missing days)116 panel = panel.ffill(limit=3)117 logger.info(118 "Built market panel for event %d: %d dates x %d markets",119 event_id, len(panel), len(panel.columns),120 )121 return panel122123124def build_arrow_debreu_state_prices(event_id: int = FED_JUNE_2026_EVENT_ID) -> pd.DataFrame:125 """Build Arrow-Debreu state-price matrix for the Fed June 2026 neg-risk group.126127 Each market corresponds to one Arrow-Debreu state128 (50+ bps decrease, 25 bps decrease, no change, 25 bps increase, 50+ bps increase).129 The state price equals the market YES price.130131 Adds:132 ``state_price_sum`` -- sum of state prices per day (should be close to 1.0).133 ``completeness_deviation`` -- ``state_price_sum - 1.0``.134 """135 panel = build_market_panel(event_id)136 if panel.empty:137 return panel138139 # Canonical ordering of the state space140 canonical_order = [141 "50+ bps decrease",142 "25 bps decrease",143 "No change",144 "25 bps increase",145 "50+ bps increase",146 ]147 # Reorder columns to canonical order, keeping only those present148 cols = [c for c in canonical_order if c in panel.columns]149 panel = panel[cols]150151 panel["state_price_sum"] = panel[cols].sum(axis=1)152 panel["completeness_deviation"] = panel["state_price_sum"] - 1.0153154 logger.info(155 "Arrow-Debreu state prices: mean sum = %.4f, mean |deviation| = %.4f",156 panel["state_price_sum"].mean(),157 panel["completeness_deviation"].abs().mean(),158 )159 return panel160161162def build_election_panel() -> pd.DataFrame:163 """Build panel for Election 2024 markets (Trump, Biden/Harris, etc.)."""164 return build_market_panel(ELECTION_2024_EVENT_ID)165166167# ---------------------------------------------------------------------------168# Master cleaning169# ---------------------------------------------------------------------------170171def clean_all() -> None:172 """Run all Polymarket cleaning and save Parquet files to ``data/processed/``."""173 DATA_PROCESSED.mkdir(parents=True, exist_ok=True)174175 # 1. Fed June 2026 -- Arrow-Debreu state prices176 fed_panel = build_arrow_debreu_state_prices()177 if not fed_panel.empty:178 out = DATA_PROCESSED / "fed_june2026_state_prices.parquet"179 fed_panel.to_parquet(out)180 logger.info("Saved %s (%d rows)", out.name, len(fed_panel))181182 # 2. Fed June 2026 -- raw market panel (without AD columns)183 fed_raw = build_market_panel(FED_JUNE_2026_EVENT_ID)184 if not fed_raw.empty:185 out = DATA_PROCESSED / "fed_june2026_panel.parquet"186 fed_raw.to_parquet(out)187 logger.info("Saved %s (%d rows)", out.name, len(fed_raw))188189 # 3. Election 2024 panel190 election = build_election_panel()191 if not election.empty:192 out = DATA_PROCESSED / "election_2024_panel.parquet"193 election.to_parquet(out)194 logger.info("Saved %s (%d rows)", out.name, len(election))195196 logger.info("=== Polymarket cleaning complete ===")197