spb/wp47_uqo

1"""Independent cross-check of the intraday factors with Treasury ETFs (SHY, IEI, IEF, TLH, TLT), 09:30–16:00 ET, 2011–2026.2Construction is deliberately different from the futures one: no daily regression. Each ETF minute return r_i (percent) is converted to an3implied yield change at the fund's approximate maturity point, dy_i = -r_i / D_i (percentage points), with modified durations4D = {SHY 1.9, IEI 4.4, IEF 7.6, TLH 11.0, TLT 17.0} years and maturity points tau = {1.9, 4.5, 8.5, 15, 25} years (approximate, fixed);5then the three NS factor changes are the OLS fit of the five dy_i on the NS loadings, minute by minute. Missing minutes (no trade) are6zero returns once the fund has traded that day. Agreement with the futures-based factors is measured by correlation and slope at71/5/30-minute sampling (09:35–15:55 ET), by period, and on FOMC windows."""8import sys, json9from pathlib import Path10import numpy as np, pandas as pd, polars as pl11ROOT = Path(__file__).resolve().parents[1]; sys.path.insert(0, str(ROOT / "src"))12from wp47.ns import loadings, LAMBDA_DL13CACHE = ROOT / "data/cache"; OUT = ROOT / "results/metrics"14ETFS = ["SHY", "IEI", "IEF", "TLH", "TLT"]; DUR = np.array([1.9, 4.4, 7.6, 11.0, 17.0]); TAU = np.array([1.9, 4.5, 8.5, 15.0, 25.0])15df = pl.concat([pl.read_parquet(f, columns=["symbol_as_reported", "timestamp_utc", "close"]) for f in sorted((CACHE / "etf_1m").glob("*.parquet"))])16df = df.sort(["symbol_as_reported", "timestamp_utc"]).unique(subset=["symbol_as_reported", "timestamp_utc"], keep="last", maintain_order=True)17pdf = df.to_pandas(); pdf["ts"] = pd.to_datetime(pdf["timestamp_utc"], utc=True).dt.tz_convert("America/New_York")18mod = pdf.ts.dt.hour * 60 + pdf.ts.dt.minute19pdf = pdf[(mod >= 570) & (mod < 960)]20wide = pdf.pivot(index="ts", columns="symbol_as_reported", values="close")[ETFS].sort_index()21# complete minute grid per day, forward-fill prices within the day22days = wide.index.normalize().unique()23grid = pd.DatetimeIndex(np.concatenate([pd.date_range(d + pd.Timedelta(hours=9, minutes=30), d + pd.Timedelta(hours=15, minutes=59), freq="1min").values for d in days])).tz_localize("UTC").tz_convert("America/New_York") if wide.index.tz is None else None24if grid is None:25    grid = pd.DatetimeIndex(np.concatenate([pd.date_range(d + pd.Timedelta(hours=9, minutes=30), d + pd.Timedelta(hours=15, minutes=59), freq="1min").tz_localize(None).values for d in days.tz_localize(None)])).tz_localize("America/New_York", ambiguous="NaT", nonexistent="NaT")26    grid = grid[~grid.isna()]27W = wide.reindex(grid)28day = pd.Series(W.index.normalize(), index=W.index)29Wf = W.groupby(day.values).ffill()30r = 100 * np.log(Wf).diff(); r[(day != day.shift()).values] = np.nan31r = r.where(r.abs() < 3)32dy = -r / DUR  # implied yield changes (pp) at TAU33X = loadings(TAU, LAMBDA_DL); Pinv = np.linalg.pinv(X)34ok = dy.notna().all(axis=1)35B = (dy[ok].values @ Pinv.T)36E = pd.DataFrame(B, index=dy.index[ok], columns=["dL", "dS", "dC"]); E.to_parquet(CACHE / "factors_1m_etf.parquet")37Fm = pd.read_parquet(CACHE / "factors_1m.parquet").set_index("ts"); Fm.index = pd.to_datetime(Fm.index, utc=True).tz_convert("America/New_York")38Fm = Fm[["dL", "dS", "dC"]].dropna()39J = E.join(Fm, how="inner", lsuffix="_etf", rsuffix="_fut")40res = {"n_minutes_overlap": int(len(J)), "first": str(J.index.min()), "last": str(J.index.max())}41pers = {"2011_2015": ("2011", "2015"), "2016_2019": ("2016", "2019"), "2020_2021": ("2020", "2021"), "2022_2026": ("2022", "2026")}42for k in (1, 5, 30):43    Jk = J if k == 1 else J.groupby(J.index.floor(f"{k}min")).sum()44    m = Jk.index.hour * 60 + Jk.index.minute; Jk = Jk[(m >= 575) & (m < 955)]45    res[f"corr_{k}m"] = {c: round(float(Jk[f"{c}_etf"].corr(Jk[f"{c}_fut"])), 4) for c in ("dL", "dS", "dC")}46    res[f"beta_fut_on_etf_{k}m"] = {c: round(float(np.polyfit(Jk[f"{c}_etf"], Jk[f"{c}_fut"], 1)[0]), 4) for c in ("dL", "dS", "dC")}47    res[f"corr_{k}m_by_period"] = {per: {c: round(float(Jk.loc[a:b, f"{c}_etf"].corr(Jk.loc[a:b, f"{c}_fut"])), 4) for c in ("dL", "dS", "dC")} for per, (a, b) in pers.items()}48    res[f"sd_bp_{k}m"] = {c: {"etf": round(float(Jk[f"{c}_etf"].std() * 100), 3), "fut": round(float(Jk[f"{c}_fut"].std() * 100), 3)} for c in ("dL", "dS", "dC")}49ev = pd.read_csv(OUT / "events.csv", parse_dates=["date"]); ev["ts"] = pd.to_datetime(ev["ts"], utc=True).dt.tz_convert("America/New_York")50rows = []51for _, e in ev[(ev.type == "FOMC") & (ev.scheduled) & (ev.ts >= J.index.min())].iterrows():52    s = J.loc[e.ts - pd.Timedelta(minutes=5): e.ts + pd.Timedelta(minutes=24)]53    if len(s) >= 24: rows.append(s.sum())54Wd = pd.DataFrame(rows)55res["fomc_window_corr_etf_vs_fut"] = {c: round(float(Wd[f"{c}_etf"].corr(Wd[f"{c}_fut"])), 4) for c in ("dL", "dS", "dC")}56res["fomc_window_beta_fut_on_etf"] = {c: round(float(np.polyfit(Wd[f"{c}_etf"], Wd[f"{c}_fut"], 1)[0]), 4) for c in ("dL", "dS", "dC")}57res["fomc_windows_n"] = int(len(Wd))58# daily cross-check: regular-hours cumulated ETF factors vs futures factors over the same minutes59Dd = J.groupby(J.index.normalize()).sum()60res["daily_regular_hours_corr"] = {c: round(float(Dd[f"{c}_etf"].corr(Dd[f"{c}_fut"])), 4) for c in ("dL", "dS", "dC")}61json.dump(res, open(OUT / "etf_crossval.json", "w"), indent=1); print(json.dumps({k: v for k, v in res.items() if "by_period" not in k}, indent=1)); print(res["corr_5m_by_period"])62