spb/wp47_uqo

1"""Quick data-quality scan of the cached 1-minute futures: coverage by year, session structure, gap/jump diagnostics."""2import sys, json3from pathlib import Path4import polars as pl5CACHE = Path(__file__).resolve().parents[1] / "data" / "cache"6out = {}7for root in ["ZT", "ZF", "ZN", "TN", "US", "UB"]:8    df = pl.concat([pl.read_parquet(f, columns=["timestamp_utc", "session_date", "open", "close", "volume"]) for f in sorted((CACHE / "futures_1m").glob(f"{root}_*.parquet")) if f.stat().st_size > 5000])9    df = df.sort("timestamp_utc").unique(subset=["timestamp_utc"], keep="last")10    df = df.with_columns(pl.col("timestamp_utc").dt.convert_time_zone("America/New_York").alias("ts_et"))11    df = df.with_columns(pl.col("ts_et").dt.year().alias("year"), pl.col("ts_et").dt.hour().alias("hour"))12    per_year = df.group_by("year").agg(pl.len().alias("bars"), pl.col("session_date").n_unique().alias("sessions"), pl.col("volume").sum().alias("vol")).sort("year")13    # session boundary: first bar after 17:00 ET; check first-return jump (roll suspicion)14    d = df.with_columns(pl.col("close").log().diff().alias("r"))15    big = d.filter(pl.col("r").abs() > 0.004).select(pl.len()).item()16    hours = df.group_by("hour").agg(pl.len().alias("n")).sort("hour")17    out[root] = {"rows": df.height, "first": str(df["ts_et"].min()), "last": str(df["ts_et"].max()),18                 "per_year": per_year.to_dicts(), "n_abs_r_gt_40bp": big,19                 "bars_by_hour_et": {int(h): int(n) for h, n in zip(hours["hour"], hours["n"])}}20    print(root, df.height, str(df["ts_et"].min())[:10], str(df["ts_et"].max())[:10], "big jumps:", big, flush=True)21    print("  sessions/yr:", {r["year"]: r["sessions"] for r in per_year.to_dicts()})22json.dump(out, open(CACHE / "eda_futures.json", "w"), indent=1, default=str)23