spb/hfmarketdata

1#!/usr/bin/env python32"""Stitch a continuous futures series locally from individual contracts (transparent methodology,3and a fallback when the v2 /continuous endpoint is unavailable).45  stitch_local.py --root CL --start 2023-01-01 [--roll volume|open_interest|calendar] [--roll-days 5]6                  [--adjust none|back_adjusted|ratio] [--out cl_local.csv]78Costs one request per contract in the range (≈ 12/yr for CL, 4/yr for ES) — set HFMD_API_KEY.9"""10from __future__ import annotations1112import argparse13import sys14from pathlib import Path1516import numpy as np17import pandas as pd1819sys.path.insert(0, str(Path(__file__).parent))20import hfmd  # noqa: E402212223def main() -> int:24    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)25    ap.add_argument("--root", required=True)26    ap.add_argument("--start", default="2023-01-01")27    ap.add_argument("--end")28    ap.add_argument("--roll", default="volume", choices=["volume", "open_interest", "calendar"])29    ap.add_argument("--roll-days", type=int, default=5, help="calendar rule: business days before expiration")30    ap.add_argument("--adjust", default="ratio", choices=["none", "back_adjusted", "ratio"])31    ap.add_argument("--out")32    a = ap.parse_args()33    root = a.root.upper()3435    cons = hfmd.contracts(root)36    if cons.empty:37        print("no contracts returned", file=sys.stderr)38        return 139    cons["expiration_date"] = pd.to_datetime(cons["expiration_date"])40    cons["last_data_date"] = pd.to_datetime(cons.get("last_data_date"))41    lo = pd.Timestamp(a.start) - pd.Timedelta(days=120)42    hi = pd.Timestamp(a.end) if a.end else pd.Timestamp.today()43    sel = cons[(cons["expiration_date"] >= lo) & (cons["first_data_date"].pipe(pd.to_datetime) <= hi)].sort_values("expiration_date") if "first_data_date" in cons else cons[cons["expiration_date"] >= lo].sort_values("expiration_date")44    print(f"{root}: {len(sel)} contracts in scope ({sel['symbol'].iloc[0]} … {sel['symbol'].iloc[-1]})")4546    panels: dict[str, pd.DataFrame] = {}47    for sym in sel["symbol"]:48        df = hfmd.contract_bars(sym, "1day", a.start and str((pd.Timestamp(a.start) - pd.Timedelta(days=120)).date()), a.end)49        if df.empty:50            hfmd.log(f"  {sym}: no bars (gap reported, not filled)")51            continue52        panels[sym] = df.set_index("datetime")53    if not panels:54        return 155    order = [s for s in sel["symbol"] if s in panels]56    expiry = dict(zip(sel["symbol"], sel["expiration_date"]))5758    # --- choose the active contract on each date ---------------------------------------------------59    dates = sorted(set().union(*[p.index for p in panels.values()]))60    dates = [d for d in dates if d >= pd.Timestamp(a.start)]61    active_idx, active, rolls = 0, order[0], []62    chosen: list[tuple[pd.Timestamp, str]] = []63    for d in dates:64        # skip contracts already expired65        while active_idx + 1 < len(order) and expiry[active] < d:66            active_idx += 167            active = order[active_idx]68            rolls.append((d, active, "expired"))69        if active_idx + 1 < len(order):70            nxt = order[active_idx + 1]71            switch = False72            if a.roll == "calendar":73                switch = d >= expiry[active] - pd.tseries.offsets.BDay(a.roll_days)74            elif d in panels[active].index and d in panels[nxt].index:75                col = "volume" if a.roll == "volume" else "open_interest"76                if col in panels[nxt].columns and col in panels[active].columns:77                    switch = float(panels[nxt].loc[d, col] or 0) > float(panels[active].loc[d, col] or 0)78            if switch:79                active_idx += 180                active = nxt81                rolls.append((d, active, a.roll))82        chosen.append((d, active))8384    # --- assemble + adjust --------------------------------------------------------------------------85    rows = []86    for d, sym in chosen:87        if d in panels[sym].index:88            r = panels[sym].loc[d]89            rows.append({"datetime": d, "contract": sym, "open": r["open"], "high": r["high"], "low": r["low"], "close": r["close"], "volume": r.get("volume"), "open_interest": r.get("open_interest")})90        else:91            rows.append({"datetime": d, "contract": sym, "open": np.nan, "high": np.nan, "low": np.nan, "close": np.nan, "volume": np.nan, "open_interest": np.nan})92    out = pd.DataFrame(rows).set_index("datetime")9394    if a.adjust != "none":95        adj = out.copy()96        # walk backwards: at each roll, compute the gap between the new and old contract on the roll date97        roll_dates = [d for d, _, _ in rolls if d in out.index]98        for d in reversed(roll_dates):99            i = out.index.get_loc(d)100            new_sym, old_sym = out["contract"].iloc[i], out["contract"].iloc[i - 1] if i > 0 else None101            if not old_sym or d not in panels[old_sym].index or d not in panels[new_sym].index:102                continue103            new_c, old_c = float(panels[new_sym].loc[d, "close"]), float(panels[old_sym].loc[d, "close"])104            before = adj.index < d105            for col in ("open", "high", "low", "close"):106                if a.adjust == "back_adjusted":107                    adj.loc[before, col] = adj.loc[before, col] + (new_c - old_c)108                else:109                    adj.loc[before, col] = adj.loc[before, col] * (new_c / old_c if old_c else 1.0)110        out = adj111112    print(f"\nrolls ({a.roll}{' ' + str(a.roll_days) + 'bd' if a.roll == 'calendar' else ''}, adjust={a.adjust}): {len(rolls)}")113    for d, sym, why in rolls:114        print(f"  {d.date()}  → {sym}  ({why})")115    print(f"\n{len(out):,} rows {out.index[0].date()} → {out.index[-1].date()} · missing closes: {int(out['close'].isna().sum())}")116    print(f"last close (adjusted={a.adjust}): {out['close'].iloc[-1]:.4f} · contract {out['contract'].iloc[-1]}")117    if a.out:118        out.to_csv(a.out)119        print(f"wrote {a.out}")120    return 0121122123if __name__ == "__main__":124    sys.exit(main())125