"""Integration tests — backfill + the 7 `/v1/futures/*` endpoints on the synthetic lake.""" from __future__ import annotations import io import os import subprocess import sys from datetime import date from pathlib import Path import pandas as pd import pytest TODAY = date(2025, 7, 1) # fixture lake ends 2025-06-30 @pytest.fixture(scope="module") def backfilled(app, lake): from futures.backfill import run_backfill s1 = run_backfill(today=TODAY) s2 = run_backfill(today=TODAY) # idempotent assert s1["contracts"] == s2["contracts"] > 0 return s1 # ---- backfill ------------------------------------------------------------------------------------------------ def test_backfill_summary_and_tables(backfilled, app): from core.db import session from futures.models import FuturesContract, FuturesRoot from sqlalchemy import select assert backfilled["rule_based"] == backfilled["contracts"] # ES, CL, NG, E6 all have exchange rules assert backfilled["active"] > 0 and backfilled["seconds"] < 30 with session() as s: es = s.execute(select(FuturesContract).where(FuturesContract.symbol == "ESH25")).scalar_one() assert es.expiration_date == date(2025, 3, 21) and es.expiration_source == "rule" assert es.first_data_date is not None and es.last_data_date == date(2025, 3, 21) assert es.status == "expired" and es.volume_avg_daily > 0 and es.open_interest_last > 0 cl = s.execute(select(FuturesContract).where(FuturesContract.symbol == "CLJ25")).scalar_one() assert cl.first_notice_date == date(2025, 3, 21) and cl.expiration_date == date(2025, 3, 20) root = s.get(FuturesRoot, "ES") assert root.source == "reference" and root.contracts_count >= 14 and root.exchange == "CME" zk = s.get(FuturesRoot, "ZK") # from meta csv only → derived assert zk.source == "derived" and zk.name == "Unknown Product" and zk.exchange == "XXX" and zk.expiry_rule == "data" e6 = s.get(FuturesRoot, "E6") assert '"6E"' in e6.aliases def test_backfill_partial_roots(backfilled, app): from futures.backfill import run_backfill s = run_backfill(roots=["6E"], today=TODAY) # alias accepted through normalize in CLI; here lake root E6 expected assert s["contracts"] == 0 # "6E" is not a lake root: nothing scanned… s = run_backfill(roots=["E6"], today=TODAY) assert s["contracts"] >= 8 and s["roots"] == 1 def test_backfill_cli(lake): env = {**os.environ, "HFMD_DATA_ROOT": str(lake), "HFMD_STATE_DB": str(lake / "state" / "hfmd.db"), "HFMD_ENV": "test"} script = Path(__file__).resolve().parents[1] / "scripts" / "backfill_contracts.py" r = subprocess.run([sys.executable, str(script), "--roots", "6E,ES", "--today", str(TODAY), "--json"], env=env, capture_output=True, text=True, timeout=120, check=False) assert r.returncode == 0, r.stderr import json out = json.loads(r.stdout.strip().splitlines()[-1]) assert out["contracts"] >= 20 and out["rule_based"] == out["contracts"] # ---- roots / contracts ---------------------------------------------------------------------------------------- def test_roots(client, backfilled): r = client.get("/v1/futures/roots") assert r.status_code == 200 body = r.json() roots = {x["root"]: x for x in body["data"]} assert body["meta"]["count"] == len(roots) >= 120 assert roots["ES"]["name"] == "E-mini S&P 500" and roots["ES"]["tick_value"] == 12.5 and roots["ES"]["expiry_rule"] == "third_friday" assert roots["ES"]["contracts_count"] >= 14 and roots["ES"]["first_data_date"] and roots["ES"]["source"] == "reference" assert roots["E6"]["aliases"] == ["6E"] and roots["US"]["aliases"] == ["ZB"] assert roots["ZK"]["source"] == "derived" and roots["ZK"]["asset_class"] is None r = client.get("/v1/futures/roots?asset_class=energy&search=crude") assert {x["root"] for x in r.json()["data"]} == {"CL", "MCL", "B"} r = client.get("/v1/futures/roots?exchange=NYMEX") assert {x["root"] for x in r.json()["data"]} >= {"CL", "BZ", "NG", "HO", "RB"} r = client.get("/v1/futures/roots?format=csv") assert r.status_code == 200 and r.text.startswith("root,name,exchange") def test_contracts_list_filters_and_sort(client, backfilled): r = client.get("/v1/futures/ES/contracts") assert r.status_code == 200 rows = r.json()["data"] exps = [x["expiration_date"] for x in rows] assert exps == sorted(exps) and len(rows) >= 14 assert rows[0]["expiration_source"] == "rule" and rows[0]["timeframes"]["1day"]["rows"] > 0 assert rows[0]["files"]["1day"][0].endswith("_1day.parquet") r = client.get("/v1/futures/es/contracts?status=active&sort=-expiration_date") act = r.json()["data"] assert {x["status"] for x in act} == {"active"} and act[0]["expiration_date"] > act[-1]["expiration_date"] assert {x["symbol"] for x in act} == {"ESU25", "ESZ25", "ESH26", "ESM26"} r = client.get("/v1/futures/ES/contracts?from=2024-01-01&to=2024-12-31") assert {x["symbol"] for x in r.json()["data"]} == {"ESH24", "ESM24", "ESU24", "ESZ24"} r = client.get("/v1/futures/6E/contracts") assert r.status_code == 200 and r.json()["meta"]["root"] == "E6" and r.json()["meta"]["count"] >= 8 assert client.get("/v1/futures/ES/contracts?sort=bogus").json()["error"]["code"] == "INVALID_PARAMETER" assert client.get("/v1/futures/ES/contracts?status=maybe").json()["error"]["code"] == "INVALID_PARAMETER" r = client.get("/v1/futures/ES/contracts?format=csv") assert r.status_code == 200 and r.headers["X-Row-Count"] == str(len(rows)) # ---- bars ------------------------------------------------------------------------------------------------------ def test_contract_bars_daily_json(client, backfilled): r = client.get("/v1/futures/contract/ESH25/bars?interval=1d&from=2025-01-01&to=2025-01-10") assert r.status_code == 200 body = r.json() assert body["meta"]["symbol"] == "ESH25" and body["meta"]["interval"] == "1d" and body["meta"]["next_cursor"] is None assert body["meta"]["expiration_date"] == "2025-03-21" and body["meta"]["expiration_source"] == "rule" dates = [x["datetime"] for x in body["data"]] assert dates == sorted(dates) and dates[0] == "2025-01-01" and dates[-1] == "2025-01-10" # fixture = plain weekdays assert set(body["data"][0]) == {"symbol", "datetime", "open", "high", "low", "close", "volume", "open_interest"} assert r.headers["X-Row-Count"] == str(len(dates)) def test_contract_bars_intraday_utc_and_sessions(client, backfilled): # November 2024 (EST, UTC-5): RTH starts 09:30 ET = 14:30Z; ETH block 08:00 ET = 13:00Z r = client.get("/v1/futures/contract/es_h25/bars?interval=1m&session=rth&limit=2") d = r.json()["data"] assert d[0]["datetime"].endswith("T14:30:00Z") and d[1]["datetime"].endswith("T14:31:00Z") assert "open_interest" not in d[0] r = client.get("/v1/futures/contract/ESH2025/bars?interval=1m&session=eth&limit=1") assert r.json()["data"][0]["datetime"].endswith("T13:00:00Z") r = client.get("/v1/futures/contract/ES_H25/bars?interval=1m&session=all&limit=1") assert r.json()["data"][0]["datetime"].endswith("T13:00:00Z") # June 2025 (EDT, UTC-4): 09:30 ET = 13:30Z — DST handled r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&session=rth&from=2025-06-16&limit=1") assert r.json()["data"][0]["datetime"] == "2025-06-16T13:30:00Z" # timeframe alias and 1hour interval r = client.get("/v1/futures/contract/ESU25/bars?timeframe=1hour&limit=1&from=2025-06-16") assert r.status_code == 200 and r.json()["meta"]["interval"] == "1h" # FX root uses its own RTH window (08:20 ET → 13:20Z in December) r = client.get("/v1/futures/contract/6EZ25/bars?interval=1m&session=rth&limit=1") assert r.json()["meta"]["symbol"] == "E6Z25" and r.json()["data"][0]["datetime"].endswith("T13:20:00Z") def test_contract_bars_from_to_datetime_bounds(client, backfilled): r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&from=2025-06-16T13:30:00Z&to=2025-06-16T13:35:00Z") d = r.json()["data"] assert [x["datetime"] for x in d] == [f"2025-06-16T13:3{i}:00Z" for i in range(5)] r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&from=not-a-date") assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER" def test_cursor_pagination(client_hu, backfilled): r1 = client_hu.get("/v1/futures/contract/ESH25/bars?interval=5m&limit=7") b1 = r1.json() assert len(b1["data"]) == 7 and b1["meta"]["next_cursor"] r2 = client_hu.get(f"/v1/futures/contract/ESH25/bars?interval=5m&limit=7&cursor={b1['meta']['next_cursor']}") b2 = r2.json() assert len(b2["data"]) == 7 and b2["data"][0]["datetime"] > b1["data"][-1]["datetime"] full = client_hu.get("/v1/futures/contract/ESH25/bars?interval=5m&limit=14").json()["data"] assert [x["datetime"] for x in b1["data"] + b2["data"]] == [x["datetime"] for x in full] last = client_hu.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=100000").json() assert last["meta"]["next_cursor"] is None assert client_hu.get("/v1/futures/contract/ESH25/bars?cursor=%%%").json()["error"]["code"] == "INVALID_PARAMETER" assert client_hu.get("/v1/futures/contract/ESH25/bars?limit=0").json()["error"]["code"] == "INVALID_PARAMETER" def test_bars_csv_and_parquet(client, backfilled): r = client.get("/v1/futures/contract/ESH25/bars?interval=1m&limit=3&format=csv") assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv") and r.headers["X-Row-Count"] == "3" lines = r.text.strip().splitlines() assert lines[0] == "symbol,datetime,open,high,low,close,volume" and lines[1].split(",")[1].endswith("Z") r = client.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=2&format=csv") assert r.text.splitlines()[1].split(",")[1] == "2024-02-15" # daily = plain date r = client.get("/v1/futures/contract/ESH25/bars?interval=1m&limit=50&format=parquet") assert r.status_code == 200 and r.headers["content-type"] == "application/vnd.apache.parquet" and r.headers["X-Row-Count"] == "50" df = pd.read_parquet(io.BytesIO(r.content)) assert len(df) == 50 and str(df["datetime"].dtype).endswith("UTC]") and list(df.columns)[:2] == ["symbol", "datetime"] def test_bars_merge_archive_and_update_dedup(client_hu, backfilled): """ESH25 exists in both buckets (archive ≤ 2024-12-31, update full): no duplicate dates.""" d = client_hu.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=100000").json()["data"] dates = [x["datetime"] for x in d] assert len(dates) == len(set(dates)) and dates[0] < "2024-12-31" < dates[-1] # ---- coverage ------------------------------------------------------------------------------------------------ def test_coverage(client, backfilled): r = client.get("/v1/futures/contract/ESH25/coverage") assert r.status_code == 200 c = r.json()["data"] assert c["symbol"] == "ESH25" and c["expiration_source"] == "rule" and c["expiration_date"] == "2025-03-21" assert c["intervals"]["1d"]["available"] and c["intervals"]["1d"]["open_interest"] is True and c["intervals"]["1d"]["rows"] > 200 assert c["intervals"]["1m"]["available"] and c["intervals"]["1m"]["open_interest"] is False assert set(c["intervals"]["1m"]["files"]) == {"archive", "update"} assert c["gaps"] == [] and isinstance(c["notes"], list) # ---- chain / term structure ------------------------------------------------------------------------------------ def test_chain(client, backfilled): r = client.get("/v1/futures/ES/chain?as_of=2024-09-10") assert r.status_code == 200 body = r.json() rows = body["data"] assert body["meta"]["front"] == "ESU24" and rows[0]["position"] == 1 and rows[0]["days_to_expiry"] == 10 assert [x["symbol"] for x in rows] == ["ESU24", "ESZ24", "ESH25", "ESM25", "ESU25"] assert rows[0]["close"] and rows[0]["last_date"] == "2024-09-10" and rows[0]["open_interest"] > 0 assert all(x["expiration_date"] >= "2024-09-10" for x in rows) r = client.get("/v1/futures/ES/chain?as_of=2010-01-01") assert r.json()["data"] == [] and r.json()["meta"]["note"] assert client.get("/v1/futures/ES/chain?as_of=yesterday").json()["error"]["code"] == "INVALID_PARAMETER" assert client.get("/v1/futures/ES/chain?as_of=2024-09-10&format=csv").status_code == 200 def test_term_structure(client, backfilled): r = client.get("/v1/futures/cl/term-structure?as_of=2025-06-30") assert r.status_code == 200 body = r.json() rows, meta = body["data"], body["meta"] assert meta["root"] == "CL" and meta["structure"] in ("contango", "backwardation", "flat") assert rows[0]["slope_annualized"] is None and rows[0]["settle"] == meta["front_settle"] assert rows[1]["slope_annualized"] is not None and rows[1]["spread_vs_front"] == pytest.approx(rows[1]["settle"] - rows[0]["settle"]) dte = [x["days_to_expiry"] for x in rows] assert dte == sorted(dte) and dte[0] >= 0 if meta["structure"] == "contango": assert rows[1]["settle"] > rows[0]["settle"] # ---- continuous ------------------------------------------------------------------------------------------------ def test_continuous_calendar_roll_and_back_adjustment(client, backfilled): r = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=none&from=2024-03-01&to=2024-03-31") assert r.status_code == 200 raw = r.json() syms = [x["symbol"] for x in raw["data"]] assert syms[0] == "ESH24" and syms[-1] == "ESM24" and syms == sorted(syms, key=syms.index) assert raw["meta"]["roll_dates"] == [pytest.approx(raw["meta"]["roll_dates"][0])] rd = raw["meta"]["roll_dates"][0] assert rd["date"] == "2024-03-18" and rd["from_symbol"] == "ESH24" and rd["to_symbol"] == "ESM24" and rd["adjusted"] assert rd["gap_session"] == "2024-03-15" and rd["gap"] == pytest.approx(rd["ratio"] * 0 + rd["gap"]) # switch day: last ESH24 bar is its expiry (Fri 15), first ESM24 bar is Mon 18 last_h = max(x["datetime"] for x in raw["data"] if x["symbol"] == "ESH24") first_m = min(x["datetime"] for x in raw["data"] if x["symbol"] == "ESM24") assert last_h == "2024-03-15" and first_m == "2024-03-18" adj = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=back_adjusted&from=2024-03-01&to=2024-03-31").json() raw_close = {(x["symbol"], x["datetime"]): x["close"] for x in raw["data"]} offsets = {} for x in adj["data"]: offsets.setdefault(x["symbol"], set()).add(round(x["close"] - raw_close[(x["symbol"], x["datetime"])], 6)) assert len(offsets["ESH24"]) == 1 and len(offsets["ESM24"]) == 1 # constant shift per segment assert (offsets["ESH24"].pop() - offsets["ESM24"].pop()) == pytest.approx(rd["gap"], abs=1e-6) assert adj["data"][0]["volume"] == raw["data"][0]["volume"] # volume never adjusted # the last segment (latest contract) is unadjusted tail = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=back_adjusted&from=2025-06-25").json() tail_raw = client.get("/v1/futures/contract/ESU25/bars?interval=1d&from=2025-06-25").json() assert tail["meta"]["unadjusted_symbol"] == "ESM26" assert {x["symbol"] for x in tail["data"]} == {"ESU25"} or tail["data"][-1]["symbol"] == tail["meta"]["unadjusted_symbol"] if {x["symbol"] for x in tail["data"]} == {"ESU25"}: assert tail["data"][-1]["close"] != tail_raw["data"][-1]["close"] or tail["meta"]["rolls_total"] > 0 def test_continuous_volume_roll_before_expiry_and_ratio(client, backfilled): r = client.get("/v1/futures/ES/continuous?roll=volume&adjust=ratio_adjusted&from=2024-03-01&to=2024-03-31").json() rd = r["meta"]["roll_dates"] assert len(rd) == 1 and rd[0]["from_symbol"] == "ESH24" and rd[0]["to_symbol"] == "ESM24" assert "2024-03-08" < rd[0]["date"] < "2024-03-15" # volume roll a few sessions before expiry assert rd[0]["ratio"] and rd[0]["ratio"] != 1.0 assert r["meta"]["roll"] == "volume" and r["meta"]["adjust"] == "ratio_adjusted" oi = client.get("/v1/futures/ES/continuous?roll=open_interest&from=2024-03-01&to=2024-03-31").json() assert oi["meta"]["roll_dates"][0]["to_symbol"] == "ESM24" and oi["meta"]["roll_dates"][0]["date"] < "2024-03-18" def test_continuous_first_notice_roll(client, backfilled): r = client.get("/v1/futures/CL/continuous?roll=first_notice&from=2024-03-15&to=2024-03-25").json() rd = r["meta"]["roll_dates"] assert len(rd) == 1 and rd[0]["date"] == "2024-03-21" and rd[0]["from_symbol"] == "CLJ24" and rd[0]["to_symbol"] == "CLN24" def test_continuous_depth(client, backfilled): r1 = client.get("/v1/futures/ES/continuous?roll=calendar&from=2024-03-01&to=2024-03-31").json() r2 = client.get("/v1/futures/ES/continuous?roll=calendar&depth=2&from=2024-03-01&to=2024-03-31").json() r3 = client.get("/v1/futures/ES/continuous?roll=calendar&depth=3&from=2024-03-01&to=2024-03-31").json() assert [x["symbol"] for x in r2["data"]][0] == "ESM24" and r2["data"][-1]["symbol"] == "ESU24" assert r3["data"][0]["symbol"] == "ESU24" and r3["data"][-1]["symbol"] == "ESZ24" assert len(r1["data"]) == len(r2["data"]) == len(r3["data"]) assert r2["meta"]["roll_dates"][0]["from_symbol"] == "ESM24" and r2["meta"]["roll_dates"][0]["to_symbol"] == "ESU24" assert client.get("/v1/futures/ES/continuous?depth=4").json()["error"]["code"] == "INVALID_PARAMETER" def test_continuous_intraday_applies_daily_schedule(client_hu, backfilled): r = client_hu.get("/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100000") assert r.status_code == 200 body = r.json() syms = [x["symbol"] for x in body["data"]] assert set(syms) == {"ESM25", "ESU25"} and syms == sorted(syms, key=syms.index) ts = [x["datetime"] for x in body["data"]] assert ts == sorted(ts) and all(t.endswith("Z") for t in ts) and ts[0] >= "2025-06-10T13:30:00Z" assert len(body["meta"]["roll_dates"]) == 1 and body["meta"]["roll_dates"][0]["to_symbol"] == "ESU25" assert "open_interest" not in body["data"][0] page = client_hu.get("/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100").json() nxt = client_hu.get(f"/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100&cursor={page['meta']['next_cursor']}").json() assert nxt["data"][0]["datetime"] > page["data"][-1]["datetime"] r = client_hu.get("/v1/futures/ES/continuous?interval=1m&from=2025-06-10&to=2025-06-12&format=parquet") assert r.status_code == 200 and int(r.headers["X-Row-Count"]) > 0 r = client_hu.get("/v1/futures/ES/continuous?from=2025-06-10&to=2025-06-12&format=csv") assert r.text.splitlines()[0].startswith("symbol,datetime") def test_continuous_alias_root_and_empty_window(client, backfilled): r = client.get("/v1/futures/6E/continuous?roll=calendar&from=2025-06-20&to=2025-06-30") assert r.status_code == 200 and r.json()["meta"]["root"] == "E6" and r.json()["data"] r = client.get("/v1/futures/ES/continuous?from=2010-01-01&to=2010-01-31") assert r.status_code == 200 and r.json()["data"] == [] and r.json()["meta"]["roll_dates"] == [] # ---- errors ------------------------------------------------------------------------------------------------------ @pytest.mark.parametrize("url, status, code", [ ("/v1/futures/contract/BOGUS/bars", 400, "INVALID_CONTRACT_SYMBOL"), ("/v1/futures/contract/ESZ5/coverage", 400, "INVALID_CONTRACT_SYMBOL"), ("/v1/futures/contract/ESZ99/bars", 404, "CONTRACT_NOT_FOUND"), ("/v1/futures/contract/ESZ99/coverage", 404, "CONTRACT_NOT_FOUND"), ("/v1/futures/NOPE/chain", 404, "ROOT_NOT_FOUND"), ("/v1/futures/NOPE/contracts", 404, "ROOT_NOT_FOUND"), ("/v1/futures/NOPE/continuous", 404, "ROOT_NOT_FOUND"), ("/v1/futures/NOPE/term-structure", 404, "ROOT_NOT_FOUND"), ("/v1/futures/ES/continuous?roll=weird", 400, "INVALID_PARAMETER"), ("/v1/futures/ES/continuous?adjust=weird", 400, "INVALID_PARAMETER"), ("/v1/futures/contract/ESH25/bars?interval=2h", 400, "INVALID_PARAMETER"), ("/v1/futures/contract/ESH25/bars?session=lunch", 400, "INVALID_PARAMETER"), ("/v1/futures/contract/ESH25/bars?format=xml", 400, "INVALID_PARAMETER"), ]) def test_error_envelopes(client, backfilled, url, status, code): r = client.get(url) assert r.status_code == status body = r.json() assert body["error"]["code"] == code and body["error"]["docs"].endswith(f"#{code.lower()}") and body["detail"] def test_tier_row_cap_is_enforced_via_request_state(client, backfilled, app): """clamp_limit(..., request=request) honours request.state.max_rows set by a limiter middleware.""" from starlette.middleware.base import BaseHTTPMiddleware class Cap(BaseHTTPMiddleware): async def dispatch(self, request, call_next): request.state.max_rows = 10 return await call_next(request) # build a throwaway app sharing the routes from core import errors from fastapi import FastAPI from fastapi.testclient import TestClient from futures.routes import router a = FastAPI() errors.install(a) a.add_middleware(Cap) a.include_router(router) with TestClient(a) as c: r = c.get("/v1/futures/contract/ESH25/bars?limit=50") assert r.status_code == 400 and r.json()["error"]["code"] == "ROW_LIMIT_EXCEEDED" and r.json()["error"]["details"]["max_rows"] == 10 r = c.get("/v1/futures/contract/ESH25/bars?limit=5") assert r.status_code == 200 and r.headers["X-Row-Count"] == "5" # ---- OpenAPI ----------------------------------------------------------------------------------------------------- def test_openapi_documents_the_seven_endpoints(client, backfilled): spec = client.get("/openapi.json").json() paths = spec["paths"] expected = ["/v1/futures/roots", "/v1/futures/{root}/contracts", "/v1/futures/contract/{symbol}/bars", "/v1/futures/contract/{symbol}/coverage", "/v1/futures/{root}/chain", "/v1/futures/{root}/continuous", "/v1/futures/{root}/term-structure"] for p in expected: op = paths[p]["get"] assert op["tags"] == ["futures"] and op["summary"] and len(op["description"]) > 80 assert "200" in op["responses"] and "429" in op["responses"] bars = paths["/v1/futures/contract/{symbol}/bars"]["get"] assert "CONTRACT_NOT_FOUND" in bars["responses"]["404"]["description"] assert "INVALID_CONTRACT_SYMBOL" in bars["responses"]["400"]["description"] assert "example" in bars["responses"]["200"]["content"]["application/json"] names = {p["name"] for p in bars["parameters"]} assert {"interval", "from", "to", "session", "cursor", "limit", "format"} <= names