spb/hfmarketdata

1#!/usr/bin/env python32"""Backfill the futures contracts metadata (SQLite) from the Parquet lake — idempotent.34Usage (production node M3U96b):5    cd ~/hfmarketdata && HFMD_DATA_ROOT=~/firstratedata venv/bin/python scripts/backfill_contracts.py6    … --roots ES,CL,NG          only these lake roots7    … --since 2026-09-01        only contract files modified on/after this date (cron mode)8    … --verbose                 progress logs910Author: Simon-Pierre Boucher <contact@spboucher.ai>11"""12from __future__ import annotations1314import argparse15import json16import logging17import sys18from datetime import date19from pathlib import Path2021sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "hfmarketdata" / "api"))222324def main() -> int:25    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)26    ap.add_argument("--roots", help="comma-separated lake roots, e.g. ES,CL (CME aliases like 6E accepted)")27    ap.add_argument("--since", help="YYYY-MM-DD: only files modified on/after this date")28    ap.add_argument("--today", help="YYYY-MM-DD: reference date for the active/expired status (tests)")29    ap.add_argument("--verbose", "-v", action="store_true")30    ap.add_argument("--json", action="store_true", help="print the summary as JSON")31    args = ap.parse_args()32    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING, format="%(asctime)s %(levelname)s %(message)s")3334    from core.config import settings35    from futures.backfill import run_backfill36    from futures.symbols import normalize_root3738    roots = [normalize_root(r) for r in args.roots.split(",") if r.strip()] if args.roots else None39    since = date.fromisoformat(args.since) if args.since else None40    today = date.fromisoformat(args.today) if args.today else None41    if not (settings.parquet / "futures_contracts").is_dir():42        print(f"error: {settings.parquet / 'futures_contracts'} not found (set HFMD_DATA_ROOT)", file=sys.stderr)43        return 244    summary = run_backfill(roots=roots, since=since, today=today, verbose=args.verbose)45    if args.json:46        print(json.dumps(summary))47    else:48        print(f"futures backfill done in {summary['seconds']}s — {summary['contracts']} contracts "49              f"({summary['active']} active, {summary['rule_based']} rule-based expirations), "50              f"{summary['files']} files, {summary['roots']} roots, {summary['gaps']} gaps → {settings.state_db}")51    return 0525354if __name__ == "__main__":55    raise SystemExit(main())56