Repository Public
1"""Hyperliquid data layer: paging, rate-limit budget, closed bars, funding normalisation, local store.23No network: the venue is replaced by `FakeVenue`, which answers the three request types the module may send with4the limits measured on the real endpoint (most recent 5000 candles only, 500 funding rows per answer)."""5from __future__ import annotations67import json89import numpy as np10import pandas as pd11import pytest1213from spq.data import DataError, load_candles14from spq.data import hyperliquid as hl1516HOUR = 3_600_00017T0 = 1_700_000_000_000 // HOUR * HOUR181920class FakeVenue:21 def __init__(self, n_candles=12_000, n_funding=1_300, now_bar=None, candle_cap=5000, drop=(), drop_funding=()):22 self.n_candles, self.n_funding, self.cap = n_candles, n_funding, candle_cap23 self.now = T0 + (now_bar if now_bar is not None else n_candles) * HOUR - 1 # inside the last bar24 self.drop, self.drop_funding = set(drop), set(drop_funding)25 self.calls: list[dict] = []2627 def candle(self, i):28 o = 100.0 + i * 0.0129 return {"t": T0 + i * HOUR, "T": T0 + (i + 1) * HOUR - 1, "s": "BTC", "i": "1h", "o": f"{o:.2f}",30 "h": f"{o + 1:.2f}", "l": f"{o - 1:.2f}", "c": f"{o + 0.5:.2f}", "v": "3.5", "n": 10}3132 def __call__(self, body):33 self.calls.append(body)34 if body["type"] == "meta":35 return {"universe": [{"name": "BTC", "szDecimals": 5, "maxLeverage": 40},36 {"name": "OLD", "szDecimals": 1, "maxLeverage": 3, "isDelisted": True}]}37 if body["type"] == "candleSnapshot":38 r = body["req"]39 last = min(self.n_candles - 1, (self.now - T0) // HOUR)40 first_served = max(0, last - self.cap + 1) # only the most recent `cap` candles exist41 lo = max(first_served, -(-(r["startTime"] - T0) // HOUR))42 hi = min(last, (r["endTime"] - T0) // HOUR)43 return [self.candle(i) for i in range(lo, hi + 1) if i not in self.drop][: self.cap]44 if body["type"] == "fundingHistory":45 rows = [{"coin": body["coin"], "fundingRate": f"{1.25e-5 + i * 1e-9:.10f}", "premium": f"{i * 1e-7:.8f}",46 "time": T0 + i * HOUR + 12 + i % 150} for i in range(self.n_funding) if i not in self.drop_funding]47 rows = [x for x in rows if x["time"] >= body["startTime"] and x["time"] <= body.get("endTime", 1 << 62)]48 return rows[:500]49 raise AssertionError(body)505152class Clock:53 def __init__(self):54 self.t, self.slept = 0.0, []5556 def __call__(self):57 return self.t5859 def sleep(self, s):60 self.slept.append(s)61 self.t += s626364def budget():65 c = Clock()66 return hl.Budget(600, sleep=c.sleep, clock=c), c676869def test_only_the_three_public_read_only_request_types_can_be_sent():70 assert hl.ALLOWED_TYPES == ("candleSnapshot", "fundingHistory", "meta") and hl.INFO_URL.endswith("/info")71 for forbidden in ("clearinghouseState", "userFills", "order", "openOrders", None):72 with pytest.raises(DataError, match="not one of"):73 hl.post_info({"type": forbidden, "user": "0x0"}) # refused before any network access747576def test_candles_are_paged_and_only_the_most_recent_5000_exist():77 venue = FakeVenue(n_candles=12_000)78 b, _ = budget()79 raw = hl.fetch_candles("BTC", "1h", post=venue, budget=b, now_ms=venue.now)80 assert len(raw) == 5000 and raw["ts"].is_monotonic_increasing and raw["ts"].is_unique81 assert raw["ts"].iloc[0] == T0 + 7000 * HOUR and raw["ts"].iloc[-1] == T0 + 11_999 * HOUR82 assert len([c for c in venue.calls if c["type"] == "candleSnapshot"]) == 1 # one answer holds everything the venue has83 # a smaller page size forces several pages: same rows, no duplicate, no hole84 small = FakeVenue(n_candles=2_300, candle_cap=5000)85 hl_page = hl.CANDLE_PAGE86 try:87 hl.CANDLE_PAGE = 50088 small.cap = 50089 small_all = FakeVenue(n_candles=2_300, candle_cap=10**6)9091 def paged(body): # serves 500 rows per answer from the full history92 return small_all(body)[:500]9394 raw2 = hl.fetch_candles("BTC", "1h", start=pd.Timestamp(T0, unit="ms", tz="UTC"), post=paged, budget=budget()[0],95 now_ms=small_all.now)96 finally:97 hl.CANDLE_PAGE = hl_page98 assert len(raw2) == 2300 and np.array_equal(np.diff(raw2["ts"]), np.full(2299, HOUR))99100101def test_download_keeps_closed_bars_merges_and_reports_quality(tmp_path):102 venue = FakeVenue(n_candles=300, n_funding=301)103 b, _ = budget()104 # as_of is "now" in download(): every fake bar is long closed; the bar in progress is tested below105 out = hl.download("BTC", "1h", out=tmp_path, post=venue, budget=b)106 assert out["ok"] and out["candles"]["n_bars"] == 300 and out["candles"]["missing_bars"] == 0107 assert out["funding"]["n"] == 301 and out["funding"]["max_offset_ms"] <= 161 # from the first candle on108 df, q = load_candles("BTC", "1h", root=tmp_path, source="hyperliquid")109 assert len(df) == 300 and q.ok() and df["open"].iloc[0] == 100.0 and str(df.index.tz) == "UTC"110 # a second download with newer bars merges without duplicates111 venue2 = FakeVenue(n_candles=320, n_funding=321)112 out = hl.download("BTC", "1h", out=tmp_path, post=venue2, budget=budget()[0])113 assert out["candles"]["n_bars"] == 320 and out["funding"]["n"] == 321114 f, fq = hl.load_funding("BTC", root=tmp_path)115 assert fq.ok() and fq.missing == 0 and len(f) == 321 and f.index[0] == pd.Timestamp(T0, unit="ms", tz="UTC")116117118def test_the_bar_in_progress_is_dropped_and_gaps_are_counted_never_filled():119 from spq.data.core import normalise120121 venue = FakeVenue(n_candles=50, drop={10, 11, 30})122 raw = hl.fetch_candles("BTC", "1h", post=venue, budget=budget()[0], now_ms=venue.now)123 as_of = pd.Timestamp(venue.now, unit="ms", tz="UTC") # inside bar 49124 df, q = normalise(raw, symbol="BTC", interval="1h", source="hyperliquid", as_of=as_of)125 assert q.unclosed_dropped == 1 and df.index[-1] == pd.Timestamp(T0 + 48 * HOUR, unit="ms", tz="UTC")126 assert q.missing_bars == 3 and q.max_gap_bars == 2 and len(q.gaps) == 2 and len(df) == 46127 assert not q.ok() # 3 of 49 missing is far above 0.5 %128129130def test_funding_is_paged_aligned_to_the_hour_and_checked():131 venue = FakeVenue(n_funding=1_300, drop_funding={700, 701, 702})132 raw = hl.fetch_funding("BTC", post=venue, budget=budget()[0], now_ms=T0 + 2000 * HOUR)133 assert len(raw) == 1297 and len([c for c in venue.calls if c["type"] == "fundingHistory"]) == 3134 f, q = hl.normalise_funding(raw, symbol="BTC", as_of=pd.Timestamp(T0 + 2000 * HOUR, unit="ms", tz="UTC"))135 # settlement instants are 12..161 ms after the hour: attached to the hour, the offset is reported136 assert (f.index.asi8 % (HOUR * 10**6) == 0).all() and q.max_offset_ms == 161 and q.off_grid == 0137 assert f["rate"].iloc[0] == 1.25e-5 and f["time_ms"].iloc[5] == T0 + 5 * HOUR + 17138 assert q.missing == 3 and q.max_gap == 3 and q.gaps[0][2] == 3 and q.expected == 1300 and len(f) == 1297139 assert q.ok(max_missing_fraction=0.005) and not q.ok(max_gap=2)140 # start is inclusive, end exclusive, on the settlement hour; nothing after as_of141 _, q2 = hl.normalise_funding(raw, symbol="BTC", start=pd.Timestamp(T0 + 10 * HOUR, unit="ms", tz="UTC"),142 end=pd.Timestamp(T0 + 20 * HOUR, unit="ms", tz="UTC"), as_of="2100-01-01")143 assert q2.n == 10144 _, q3 = hl.normalise_funding(raw, symbol="BTC", as_of=pd.Timestamp(T0 + 99 * HOUR + 5, unit="ms", tz="UTC"))145 assert q3.n == 100 and q3.unsettled_dropped == 1197146147148def test_funding_refuses_what_cannot_be_trusted():149 base = pd.DataFrame({"time_ms": [T0 + 10, T0 + HOUR + 10, T0 + 2 * HOUR + 10], "rate": [1e-5, 2e-5, 3e-5]})150 far = "2100-01-01"151 f, q = hl.normalise_funding(base, symbol="X", as_of=far)152 assert q.ok() and np.isnan(f["premium"]).all()153 # an instant 20 minutes after the hour cannot be attributed to a settlement154 off = base.assign(time_ms=[T0 + 10, T0 + HOUR + 20 * 60_000, T0 + 2 * HOUR + 10])155 f, q = hl.normalise_funding(off, symbol="X", as_of=far)156 assert q.off_grid == 1 and not q.ok() and len(f) == 2 and q.missing == 1157 # two rows on the same hour: conflicting -> error; identical -> merged on request158 dup = pd.concat([base, pd.DataFrame({"time_ms": [T0 + HOUR + 30], "rate": [9e-5]})])159 with pytest.raises(DataError, match="share a settlement hour"):160 hl.normalise_funding(dup, symbol="X", as_of=far)161 same = pd.concat([base, pd.DataFrame({"time_ms": [T0 + HOUR + 30], "rate": [2e-5]})])162 with pytest.raises(DataError):163 hl.normalise_funding(same, symbol="X", as_of=far)164 f, q = hl.normalise_funding(same, symbol="X", as_of=far, on_duplicate="drop_identical")165 assert q.duplicates_dropped == 1 and len(f) == 3166 bad = base.assign(rate=[1e-5, float("nan"), 3e-5])167 assert hl.normalise_funding(bad, symbol="X", as_of=far)[1].non_finite == 1168 with pytest.raises(DataError, match="time_ms and rate"):169 hl.normalise_funding(pd.DataFrame({"ts": [1]}), symbol="X")170171172def test_meta_budget_and_depth_measurement():173 venue = FakeVenue(n_candles=9000, n_funding=40)174 meta = hl.fetch_meta(post=venue, budget=budget()[0])175 assert meta["BTC"] == {"max_leverage": 40.0, "size_decimals": 5, "delisted": False} and meta["OLD"]["delisted"]176 with pytest.raises(DataError):177 hl.fetch_meta(post=lambda body: [], budget=budget()[0])178 # the budget never lets more than the limit through inside one minute179 b, clock = budget()180 for _ in range(30):181 b.take(20) # 600 of weight: fits182 assert clock.slept == []183 b.take(20)184 assert clock.slept and clock.t >= 60.0 - 1e-9 # the 31st request waited for the window to slide185 b2, clock2 = budget()186 hl.fetch_funding("BTC", post=FakeVenue(n_funding=3000), budget=b2, now_ms=T0 + 4000 * HOUR)187 spent = sum(w for _, w in b2.spent) + 0188 assert clock2.t >= 0 and spent <= 600 # 7 pages x (20 + 25) = 315 of weight189 depth = hl.measure_depth("BTC", intervals=("1h",), post=venue, budget=budget()[0])190 assert depth["candles"]["1h"]["rows_served"] == 5000 and depth["funding"]["first"].startswith(191 pd.Timestamp(T0, unit="ms", tz="UTC").isoformat()[:19])192193194def test_load_perp_universe_joins_candles_funding_and_spot_legs(tmp_path):195 for sym, n in (("BTC", 300), ("ETH", 290)):196 venue = FakeVenue(n_candles=n, n_funding=n + 1)197 hl.download(sym, "1h", out=tmp_path, post=venue, budget=budget()[0])198 # spot candles of BTC from "coinbase" (same layout as spq.data.fetch writes), with one bar missing199 spot = pd.read_parquet(tmp_path / "hyperliquid" / "BTC_1h.parquet").drop(index=[100])200 (tmp_path / "coinbase").mkdir()201 spot.to_parquet(tmp_path / "coinbase" / "BTC_1h.parquet", index=False)202 data, q, fq, raw = hl.load_perp_universe(["BTC", "ETH", "BTC.SPOT"], "1h", root=tmp_path)203 assert list(data) == ["BTC", "ETH", "BTC.SPOT"]204 idx = data["BTC"].index205 assert all(df.index.equals(idx) for df in data.values()) and len(idx) == 289 # common bars only206 assert {"funding", "premium"} <= set(data["BTC"].columns) and "funding" not in data["BTC.SPOT"].columns207 assert q["BTC.SPOT"].missing_bars == 1 and q["BTC.SPOT"].symbol == "BTC.SPOT" and fq["BTC"].ok() and set(fq) == {"BTC", "ETH"}208 # the funding of a bar is the settlement dated at its open hour209 i = 57210 assert data["BTC"]["funding"].iloc[i] == raw["BTC"]["rate"].loc[idx[i]]211 assert hl.check_funding_alignment(data, raw)["status"] == "pass"212 ref, ref_q = hl.load_spot_reference(["BTC"], "1h", idx, root=tmp_path)213 assert len(ref["BTC"]) <= 300 and ref_q["BTC"].missing_bars == 1214 assert hl.load_spot_reference(["BTC", "ETH"], "1h", idx, root=tmp_path) == ({}, {}) # no ETH spot file215 with pytest.raises(DataError, match="no funding file"):216 hl.load_funding("SOL", root=tmp_path)217 meta_file = hl.meta_path(tmp_path)218 meta_file.write_text(json.dumps({"universe": {"BTC": {"max_leverage": 40.0, "size_decimals": 5}}}))219 assert hl.load_meta(tmp_path)["universe"]["BTC"]["max_leverage"] == 40.0 and hl.load_meta(tmp_path / "none") == {}220221222def test_funding_loader_hook_replaces_the_file_reader():223 """The hook for a database table, should one exist later: any callable returning time_ms / rate / premium."""224 def from_elsewhere(symbol, start, end):225 return pd.DataFrame({"time_ms": [T0 + i * HOUR + 40 for i in range(5)], "rate": [1e-5] * 5,226 "premium": [0.0] * 5})227228 f, q = hl.load_funding("BTC", loader=from_elsewhere, as_of="2100-01-01")229 assert len(f) == 5 and q.ok()230 hl.FUNDING_LOADER = from_elsewhere231 try:232 assert len(hl.load_funding("ETH", as_of="2100-01-01")[0]) == 5233 finally:234 hl.FUNDING_LOADER = None235236237@pytest.mark.network238def test_live_hyperliquid_public_endpoint():239 """Opt-in (SPQ_NETWORK_TESTS=1): three read-only requests to the public info endpoint."""240 b = hl.Budget(300)241 meta = hl.fetch_meta(budget=b)242 assert meta["BTC"]["max_leverage"] >= 1 and meta["BTC"]["size_decimals"] >= 0243 end = pd.Timestamp.now(tz="UTC").floor("h")244 start = end - pd.Timedelta(hours=30)245 raw = hl.fetch_candles("BTC", "1h", start, end, budget=b)246 from spq.data.core import normalise247248 df, q = normalise(raw, symbol="BTC", interval="1h", source="hyperliquid", start=start, end=end)249 assert 28 <= len(df) <= 30 and q.ohlc_violations == 0 and q.off_grid == 0250 f, fq = hl.normalise_funding(hl.fetch_funding("BTC", start, end, budget=b), symbol="BTC")251 assert 28 <= len(f) <= 31 and fq.off_grid == 0 and fq.max_offset_ms < 60_000 and f["rate"].abs().max() < 0.005252