Repository Public
1"""Write a local Hyperliquid store from a JSON file, WITHOUT any network access.23 uv run python -m spq.data.hl_import --input bundle.json --out data45The API service pre-fetches the venue's candles and funding settlements itself (its read-only, rate-budgeted adapter)6and hands them over as one file; this module turns the file into exactly the layout `python -m spq.data.hyperliquid`7writes, with the same checks (`normalise`: closed bars only, grid, duplicates; settlements keyed by their instant).8The laboratory process that backtests a strategy for the API therefore never needs the network.910Input (decimal strings or numbers; times in milliseconds):1112 {"interval": "1h", "as_of_ms": 1794300000000,13 "meta": {"BTC": {"max_leverage": 40, "size_decimals": 5}},14 "candles": {"BTC": [[ts_ms, open, high, low, close, volume], ...]},15 "funding": {"BTC": [[time_ms, rate, premium or null], ...]},16 "spot": {"source": "coinbase", "candles": {"BTC": [[ts_ms, o, h, l, c, v], ...]}}} # optional1718`as_of_ms` is the instant the data were read: bars not closed at that instant are dropped. `spot` (optional) is19written as `<out>/<source>/<SYM>_<interval>.parquet` for the buy-and-hold benchmark.20"""21from __future__ import annotations2223import argparse24import json25import sys26from pathlib import Path2728import pandas as pd2930from .core import COLUMNS, DataError, normalise31from .hyperliquid import SOURCE, funding_path, meta_path, normalise_funding323334def _symbol(s) -> str:35 if not isinstance(s, str) or not (1 <= len(s) <= 12) or not s.isascii() or not s.isalnum() or s != s.upper():36 raise DataError(f"bad symbol {s!r}")37 return s383940def _candles(rows, symbol: str, interval: str, source: str, as_of) -> pd.DataFrame:41 if not isinstance(rows, list) or not rows or not all(isinstance(r, list) and len(r) == 6 for r in rows):42 raise DataError(f"{symbol}: candles must be a non-empty array of [ts_ms, open, high, low, close, volume]")43 raw = pd.DataFrame({"ts": pd.to_datetime([int(r[0]) for r in rows], unit="ms", utc=True),44 **{c: [float(r[i + 1]) for r in rows] for i, c in enumerate(COLUMNS)}})45 df, q = normalise(raw, symbol=symbol, interval=interval, source=source, as_of=as_of)46 if q.off_grid or q.ohlc_violations:47 raise DataError(f"{symbol}: {q.off_grid} timestamps off the grid, {q.ohlc_violations} impossible rows")48 return df.reset_index()495051def import_bundle(bundle: dict, out) -> dict:52 """Write the store; return a summary {symbol: {"bars", "funding"}}. Raises DataError on anything unexpected."""53 if not isinstance(bundle, dict) or not isinstance(bundle.get("candles"), dict) or not isinstance(bundle.get("funding"), dict):54 raise DataError('the bundle needs "candles" and "funding" objects')55 interval = bundle.get("interval")56 if not isinstance(interval, str):57 raise DataError('the bundle needs an "interval"')58 as_of = pd.Timestamp(int(bundle["as_of_ms"]), unit="ms", tz="UTC") if bundle.get("as_of_ms") is not None else None59 symbols = [_symbol(s) for s in bundle["candles"]]60 if sorted(symbols) != sorted(bundle["funding"]):61 raise DataError("candles and funding must cover the same symbols")62 out = Path(out)63 summary: dict = {}64 for sym in symbols:65 candles = _candles(bundle["candles"][sym], sym, interval, SOURCE, as_of)66 frows = bundle["funding"][sym]67 if not isinstance(frows, list) or not frows or not all(isinstance(r, list) and len(r) in (2, 3) for r in frows):68 raise DataError(f"{sym}: funding must be a non-empty array of [time_ms, rate, premium]")69 funding = pd.DataFrame({"time_ms": [int(r[0]) for r in frows], "rate": [float(r[1]) for r in frows],70 "premium": [float(r[2]) if len(r) > 2 and r[2] is not None else float("nan") for r in frows]})71 funding = funding.sort_values("time_ms").reset_index(drop=True)72 normalise_funding(funding, symbol=sym, as_of=as_of) # raises on two rows for one settlement hour73 cpath = out / SOURCE / f"{sym}_{interval}.parquet"74 cpath.parent.mkdir(parents=True, exist_ok=True)75 candles.to_parquet(cpath, index=False)76 fpath = funding_path(out, sym)77 fpath.parent.mkdir(parents=True, exist_ok=True)78 funding.to_parquet(fpath, index=False)79 summary[sym] = {"bars": int(len(candles)), "funding": int(len(funding))}80 meta = bundle.get("meta") or {}81 universe = {}82 for sym, m in meta.items():83 universe[_symbol(sym)] = {"max_leverage": float(m["max_leverage"]), "size_decimals": int(m["size_decimals"]),84 "delisted": False}85 mp = meta_path(out)86 mp.parent.mkdir(parents=True, exist_ok=True)87 mp.write_text(json.dumps({"fetched_at": as_of.isoformat() if as_of is not None else None, "universe": universe},88 indent=1) + "\n")89 spot = bundle.get("spot")90 if spot:91 source = spot.get("source")92 if source not in ("coinbase", "bitstamp", "kraken") or not isinstance(spot.get("candles"), dict):93 raise DataError('"spot" needs a known "source" and a "candles" object')94 for sym, rows in spot["candles"].items():95 df = _candles(rows, _symbol(sym), interval, source, as_of)96 path = out / source / f"{sym}_{interval}.parquet"97 path.parent.mkdir(parents=True, exist_ok=True)98 df.to_parquet(path, index=False)99 summary.setdefault(sym, {})["spot_bars"] = int(len(df))100 return summary101102103def main(argv=None) -> int:104 p = argparse.ArgumentParser(prog="python -m spq.data.hl_import", description=__doc__.split("\n")[0])105 p.add_argument("--input", required=True, help="JSON bundle written by the API service")106 p.add_argument("--out", required=True, help="root of the local store to write")107 a = p.parse_args(argv)108 try:109 with open(a.input, encoding="utf-8") as f:110 bundle = json.load(f)111 print(json.dumps(import_bundle(bundle, a.out)))112 except (DataError, OSError, ValueError, KeyError, TypeError) as e:113 print(f"spq.data.hl_import: {type(e).__name__}: {e}", file=sys.stderr)114 return 2115 return 0116117118if __name__ == "__main__":119 sys.exit(main())120