#!/usr/bin/env python3 """Backfill the futures contracts metadata (SQLite) from the Parquet lake — idempotent. Usage (production node M3U96b): cd ~/hfmarketdata && HFMD_DATA_ROOT=~/firstratedata venv/bin/python scripts/backfill_contracts.py … --roots ES,CL,NG only these lake roots … --since 2026-09-01 only contract files modified on/after this date (cron mode) … --verbose progress logs Author: Simon-Pierre Boucher """ from __future__ import annotations import argparse import json import logging import sys from datetime import date from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "hfmarketdata" / "api")) def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--roots", help="comma-separated lake roots, e.g. ES,CL (CME aliases like 6E accepted)") ap.add_argument("--since", help="YYYY-MM-DD: only files modified on/after this date") ap.add_argument("--today", help="YYYY-MM-DD: reference date for the active/expired status (tests)") ap.add_argument("--verbose", "-v", action="store_true") ap.add_argument("--json", action="store_true", help="print the summary as JSON") args = ap.parse_args() logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING, format="%(asctime)s %(levelname)s %(message)s") from core.config import settings from futures.backfill import run_backfill from futures.symbols import normalize_root roots = [normalize_root(r) for r in args.roots.split(",") if r.strip()] if args.roots else None since = date.fromisoformat(args.since) if args.since else None today = date.fromisoformat(args.today) if args.today else None if not (settings.parquet / "futures_contracts").is_dir(): print(f"error: {settings.parquet / 'futures_contracts'} not found (set HFMD_DATA_ROOT)", file=sys.stderr) return 2 summary = run_backfill(roots=roots, since=since, today=today, verbose=args.verbose) if args.json: print(json.dumps(summary)) else: print(f"futures backfill done in {summary['seconds']}s — {summary['contracts']} contracts " f"({summary['active']} active, {summary['rule_based']} rule-based expirations), " f"{summary['files']} files, {summary['roots']} roots, {summary['gaps']} gaps → {settings.state_db}") return 0 if __name__ == "__main__": raise SystemExit(main())