spb/wp8_uqo

1"""Clean Yahoo Finance OHLCV data and compute log returns."""23from __future__ import annotations45import numpy as np6import pandas as pd78from src.config import DATA_PROCESSED, DATA_RAW_YAHOO9from src.utils import setup_logging1011logger = setup_logging("yahoo_cleaner")121314def clean_yahoo_data() -> pd.DataFrame:15    """Load all Yahoo CSVs, compute log returns, return a combined DataFrame.1617    For each ticker file (SPY.csv, BTC_USD.csv, etc.) we:18      1. Parse dates and set DatetimeIndex.19      2. Keep Close and Volume.20      3. Compute log return = ln(Close_t / Close_{t-1}).2122    Returns a wide DataFrame with columns like ``SPY_close``, ``SPY_volume``,23    ``SPY_log_return``, etc.24    """25    combined = pd.DataFrame()2627    for csv_path in sorted(DATA_RAW_YAHOO.glob("*.csv")):28        # Skip the combined all_tickers file -- we use per-ticker files29        if csv_path.stem == "all_tickers":30            continue3132        ticker = csv_path.stem  # e.g. "SPY", "BTC_USD"33        df = pd.read_csv(csv_path, parse_dates=["date"], index_col="date")34        df.index = pd.to_datetime(df.index, utc=True).normalize().tz_localize(None)3536        # Coerce numeric37        for col in ["Close", "Volume"]:38            if col in df.columns:39                df[col] = pd.to_numeric(df[col], errors="coerce")4041        if "Close" not in df.columns:42            logger.warning("No 'Close' column in %s, skipping", csv_path.name)43            continue4445        close = df["Close"].rename(f"{ticker}_close")46        log_ret = np.log(df["Close"] / df["Close"].shift(1)).rename(f"{ticker}_log_return")4748        combined[f"{ticker}_close"] = close49        combined[f"{ticker}_log_return"] = log_ret5051        if "Volume" in df.columns:52            volume = df["Volume"].rename(f"{ticker}_volume")53            combined[f"{ticker}_volume"] = volume5455        logger.info("Loaded Yahoo ticker '%s': %d observations", ticker, len(df))5657    combined = combined.sort_index()58    logger.info(59        "Combined Yahoo data: %d rows, %d columns, date range %s to %s",60        len(combined), len(combined.columns),61        combined.index.min(), combined.index.max(),62    )63    return combined646566def clean_all() -> None:67    """Save ``financial_data.parquet`` to ``data/processed/``."""68    DATA_PROCESSED.mkdir(parents=True, exist_ok=True)6970    financial = clean_yahoo_data()71    if not financial.empty:72        out = DATA_PROCESSED / "financial_data.parquet"73        financial.to_parquet(out)74        logger.info("Saved %s (%d rows)", out.name, len(financial))7576    logger.info("=== Yahoo cleaning complete ===")77