spb/wp7_uqo Public

1# =============================================================================2# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai4# =============================================================================5"""Data access layer: processed parquets and (optional) raw DuckDB stores.67The raw stores are external to this repository and may be absent. Every8accessor that touches them raises :class:`RawDataUnavailableError` with an9actionable message instead of a bare stack trace, so downstream scripts can10skip raw-dependent sections gracefully.11"""1213from pathlib import Path1415import pandas as pd1617from . import config181920class RawDataUnavailableError(FileNotFoundError):21    """Raised when a raw DuckDB store is required but not present."""222324def _require(path: Path, hint: str) -> Path:25    if not path.exists():26        raise FileNotFoundError(27            f"Missing processed dataset: {path}\n{hint}"28        )29    return path303132# --------------------------------------------------------------------------33# Processed datasets34# --------------------------------------------------------------------------35def load_merged() -> pd.DataFrame:36    """Master analysis panel (69 tickers × ~264k ticker-days, 2010–2025)."""37    path = _require(config.MERGED_PARQUET,38                    "Run scripts/01_extract_data.py (requires the raw DuckDB stores).")39    df = pd.read_parquet(path)40    df['trade_date'] = pd.to_datetime(df['trade_date'])41    return df424344def load_realized_vol() -> pd.DataFrame:45    """Daily realized-volatility panel derived from 5-minute bars."""46    path = _require(config.REALIZED_VOL_PARQUET,47                    "Run scripts/01_extract_data.py (requires the raw DuckDB stores).")48    df = pd.read_parquet(path)49    df['trade_date'] = pd.to_datetime(df['trade_date'])50    return df515253def load_correlation_divergence() -> pd.DataFrame:54    """Implied vs realized correlation series with VIX and stress flags (RQ3)."""55    path = _require(config.CORRELATION_DIVERGENCE_PARQUET,56                    "Run scripts/04_rq3_correlation_divergence.py "57                    "(requires the raw DuckDB stores).")58    df = pd.read_parquet(path)59    df['trade_date'] = pd.to_datetime(df['trade_date'])60    return df616263def load_price_magnet() -> pd.DataFrame:64    """Filtered price-magnet observations saved by the RQ4 script."""65    path = _require(config.PRICE_MAGNET_PARQUET,66                    "Run scripts/05_rq4_greeks_decay_magnets.py "67                    "(requires the raw DuckDB stores).")68    df = pd.read_parquet(path)69    df['trade_date'] = pd.to_datetime(df['trade_date'])70    return df717273# --------------------------------------------------------------------------74# Raw DuckDB stores (optional)75# --------------------------------------------------------------------------76def raw_db_path(name: str) -> Path:77    """Absolute path of a raw store; see ``config.RAW_DB_FILES`` for names."""78    return config.DATA_RAW / config.RAW_DB_FILES[name]798081def raw_data_available(*names: str) -> bool:82    """True when every requested raw store exists on disk."""83    names = names or tuple(config.RAW_DB_FILES)84    return all(raw_db_path(n).exists() for n in names)858687def open_raw_db(name: str):88    """Open a raw DuckDB store read-only, or raise RawDataUnavailableError."""89    import duckdb9091    path = raw_db_path(name)92    if not path.exists():93        raise RawDataUnavailableError(94            f"Raw store '{config.RAW_DB_FILES[name]}' not found under {config.DATA_RAW}.\n"95            "These stores (~3.8B option records / 11.5B intraday bars) are kept outside "96            "the repository. Point WP7_RAW_DATA_DIR to the directory that contains them, "97            "or skip the raw-dependent steps — every downstream analysis runs from the "98            "processed parquets in data/processed/."99        )100    return duckdb.connect(str(path), read_only=True)101102103def load_vix_daily() -> pd.DataFrame:104    """Daily VIX close aggregated from the 5-minute index store (raw-dependent)."""105    con = open_raw_db("indices_5min")106    vix = con.execute("""107        SELECT CAST(datetime AS DATE) AS trade_date, LAST(close) AS vix_close108        FROM ohlcv WHERE symbol='VIX' GROUP BY CAST(datetime AS DATE)109        ORDER BY trade_date110    """).fetchdf()111    con.close()112    vix['trade_date'] = pd.to_datetime(vix['trade_date'])113    return vix114