spb/wp8_uqo

1"""Clean FRED macro data and build a unified event calendar."""23from __future__ import annotations45from datetime import date67import pandas as pd89from src.config import (10    CPI_RELEASE_DATES,11    DATA_PROCESSED,12    DATA_RAW_FRED,13    ELECTION_KEY_DATES,14    FOMC_DATES,15    NFP_RELEASE_DATES,16)17from src.utils import setup_logging1819logger = setup_logging("macro_cleaner")202122# ---------------------------------------------------------------------------23# Event calendar24# ---------------------------------------------------------------------------2526def build_event_calendar() -> pd.DataFrame:27    """Build a unified event calendar with columns: date, event_type, description."""28    rows: list[dict] = []2930    for d in FOMC_DATES:31        rows.append({"date": d, "event_type": "FOMC", "description": f"FOMC meeting {d}"})3233    for d in CPI_RELEASE_DATES:34        rows.append({"date": d, "event_type": "CPI", "description": f"CPI release {d}"})3536    for d in NFP_RELEASE_DATES:37        rows.append({"date": d, "event_type": "NFP", "description": f"Non-farm payrolls {d}"})3839    for label, d in ELECTION_KEY_DATES.items():40        rows.append({"date": d, "event_type": "Election", "description": label})4142    df = pd.DataFrame(rows)43    df["date"] = pd.to_datetime(df["date"])44    df = df.sort_values("date").reset_index(drop=True)45    logger.info("Built event calendar with %d events", len(df))46    return df474849# ---------------------------------------------------------------------------50# FRED data51# ---------------------------------------------------------------------------5253def clean_fred_data() -> dict[str, pd.DataFrame]:54    """Load and clean all FRED CSVs.5556    Returns a dict mapping series name to a DataFrame with DatetimeIndex.57    """58    result: dict[str, pd.DataFrame] = {}5960    for csv_path in sorted(DATA_RAW_FRED.glob("*.csv")):61        df = pd.read_csv(csv_path, parse_dates=["date"], index_col="date")62        # Coerce all value columns to numeric63        for col in df.columns:64            df[col] = pd.to_numeric(df[col], errors="coerce")65        df = df.sort_index()66        name = csv_path.stem  # e.g. "fed_funds_rate", "cpi", "unemployment"67        result[name] = df68        logger.info("Loaded FRED series '%s': %d observations, columns=%s",69                     name, len(df), list(df.columns))7071    return result727374# ---------------------------------------------------------------------------75# Master cleaning76# ---------------------------------------------------------------------------7778def clean_all() -> None:79    """Save macro_events.parquet and fred_data.parquet to ``data/processed/``."""80    DATA_PROCESSED.mkdir(parents=True, exist_ok=True)8182    # 1. Event calendar83    calendar = build_event_calendar()84    out = DATA_PROCESSED / "macro_events.parquet"85    calendar.to_parquet(out)86    logger.info("Saved %s (%d rows)", out.name, len(calendar))8788    # 2. FRED data -- merge all series into a single wide DataFrame89    fred = clean_fred_data()90    if fred:91        # Outer-join all series on date92        combined = pd.DataFrame()93        for name, df in fred.items():94            for col in df.columns:95                col_name = f"{name}_{col}" if col != name else col96                combined[col_name] = df[col]97        # Re-sort by date98        combined = combined.sort_index()99        # Forward-fill monthly series to daily grid100        combined = combined.ffill()101        out = DATA_PROCESSED / "fred_data.parquet"102        combined.to_parquet(out)103        logger.info("Saved %s (%d rows, %d columns)", out.name, len(combined), len(combined.columns))104105    logger.info("=== Macro cleaning complete ===")106