futures: lac synthétique enrichi (cycle de vie volume/OI, ETH, racine E6) et tests unitaires, intégration et performance

- make_fixtures.py : contrats ES/CL/NG/E6 avec échéances réalistes, volume/OI qui montent puis s'effondrent, bloc ETH 08:00–09:29, 30 séances intraday sur ES
- test_futures_symbols/expiry/rolls : parsing + alias, 39 dates d'échéance réelles, calendriers de roll et maths d'ajustement sur séries synthétiques
- test_futures_api : backfill → 7 endpoints → json/csv/parquet → curseur → erreurs → plafond de lignes par tier → OpenAPI
- test_futures_perf : 10 000 barres (contrat, continu, parquet) < 300 ms (marqueur slow)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 20 days ago (Sep 4, 2026) parent 9c33c9a

7 changed files +867 −19

modified pytest.ini +2 −0

		`@@ -2,6 +2,8 @@`
2	2	testpaths = tests
3	3	pythonpath = hfmarketdata/api
4	4	asyncio_mode = auto
	5	+markers =
	6	+ slow: timing-sensitive checks (deselect with -m "not slow")
5	7	filterwarnings =
6	8	ignore::DeprecationWarning
7	9	addopts = -q

modified tests/fixtures/make_fixtures.py +62 −19

		`@@ -7,12 +7,15 @@`
7	7	meta/futures/futures.csv
8	8
9	9	Every bar file carries a `ticker` column (root only for futures contracts — the contract identity is
10		−in the file name, exactly like the real lake).
	10	+in the file name, exactly like the real lake). Futures contracts get a realistic life-cycle: volume and
	11	+open interest ramp up towards expiry and collapse in the last sessions (so volume/OI rolls are testable),
	12	+daily data ends on the exchange expiry (3rd Friday for ES/E6, 20th of the preceding month for CL/NG),
	13	+intraday bars cover RTH 09:30–16:00 plus a pre-market ETH block 08:00–09:29 (Eastern, naive).
11	14	"""
12	15	from __future__ import annotations
13	16
14	17	import csv
15		−from datetime import date, datetime, timedelta
	18	+from datetime import date, timedelta
16	19	from pathlib import Path
17	20
18	21	import numpy as np
		`@@ -22,27 +25,64 @@ TIMEFRAMES = ["1min", "5min", "30min", "1hour", "1day"]`
22	25	TICKERS = {"stock": ["AAPL", "MSFT", "SMCP"], "etf": ["SPY"], "crypto": ["BTCUSD"], "index": ["SPX"], "fx": ["EURUSD"]}
23	26	ADJ = {"stock": ["adj_split", "adj_splitdiv", "UNADJUSTED"], "etf": ["adj_split", "adj_splitdiv", "UNADJUSTED"],
24	27	"crypto": ["none"], "index": ["none"], "fx": ["none"]}
25		−ROOTS = ["ES", "CL", "NG"]
	28	+ROOTS = ["ES", "CL", "NG", "E6"]
26	29	MONTHS = {"F": 1, "G": 2, "H": 3, "J": 4, "K": 5, "M": 6, "N": 7, "Q": 8, "U": 9, "V": 10, "X": 11, "Z": 12}
27		−CYCLE = {"ES": "HMUZ", "CL": "FGHJKMNQUVXZ", "NG": "FGHJKMNQUVXZ"}
	30	+CYCLE = {"ES": "HMUZ", "CL": "FJNV", "NG": "FJNV", "E6": "HMUZ"} # CL/NG: quarterly subset to keep the lake small
	31	+INTRADAY_DAYS = {"ES": 30} # sessions of intraday history per contract (default 5)
28	32
29	33
30		−def _bars(ticker: str, start: date, end: date, tf: str, seed: int, oi: bool = False) -> pd.DataFrame:
	34	+def _third_friday(y: int, m: int) -> date:
	35	+ d = date(y, m, 15)
	36	+ return d + timedelta(days=(4 - d.weekday()) % 7)
	37	+
	38	+
	39	+def contract_expiry(root: str, yy: int, month_code: str) -> date:
	40	+ y, m = 2000 + yy, MONTHS[month_code]
	41	+ if root in ("ES", "E6"):
	42	+ return _third_friday(y, m)
	43	+ pm = m - 1 or 12
	44	+ return date(y - (m == 1), pm, 20)
	45	+
	46	+
	47	+def _lifecycle(dte: np.ndarray, rng: np.random.Generator) -> tuple[np.ndarray, np.ndarray]:
	48	+ """(volume, open_interest) as a function of days to expiry: ramp up, collapse in the last sessions."""
	49	+ w = np.where(dte > 8, (150.0 - dte) / 100.0, 1.42 * (dte / 8.0) ** 2 + 0.02)
	50	+ w = np.clip(w, 0.02, None) * rng.uniform(0.97, 1.03, len(dte))
	51	+ vol = np.round(10_000 * w)
	52	+ oi = np.clip((200.0 - dte) / 150.0, 0.05, 1.0) * np.where(dte > 10, 1.0, dte / 10.0)
	53	+ oi = np.round(100_000 * oi * rng.uniform(0.98, 1.02, len(dte)))
	54	+ return vol, oi
	55	+
	56	+
	57	+def _bars(ticker: str, start: date, end: date, tf: str, seed: int, oi: bool = False, base: float = 100.0,
	58	+ expiry: date \| None = None, eth: bool = False, days: int = 5) -> pd.DataFrame:
31	59	rng = np.random.default_rng(seed)
32	60	if tf == "1day":
33	61	idx = pd.bdate_range(start, end)
34	62	else:
35	63	step = {"1min": 1, "5min": 5, "30min": 30, "1hour": 60}[tf]
36		− days = pd.bdate_range(start, end)[-5:] # intraday: last 5 sessions only, RTH 09:30-16:00 ET
37		− idx = pd.DatetimeIndex([d + timedelta(hours=9, minutes=30) + timedelta(minutes=step * i)
38		− for d in days for i in range(int(390 / step))])
	64	+ sessions = pd.bdate_range(start, end)[-days:]
	65	+ starts = [timedelta(hours=9, minutes=30)]
	66	+ counts = [int(390 / step)]
	67	+ if eth and step <= 30: # pre-market block 08:00–09:29
	68	+ starts.insert(0, timedelta(hours=8))
	69	+ counts.insert(0, int(90 / step))
	70	+ idx = pd.DatetimeIndex([d + s + timedelta(minutes=step * i) for d in sessions for s, n in zip(starts, counts) for i in range(n)])
39	71	n = len(idx)
40		− close = 100 + np.cumsum(rng.normal(0, 1, n))
	72	+ close = base + np.cumsum(rng.normal(0, 1, n))
41	73	df = pd.DataFrame({"ticker": ticker, "datetime": idx, "open": close + rng.normal(0, .2, n),
42	74	"high": close + abs(rng.normal(0, .5, n)), "low": close - abs(rng.normal(0, .5, n)),
43		− "close": close, "volume": rng.integers(100, 10_000, n).astype(float)})
44		− if oi:
45		− df["open_interest"] = rng.integers(1_000, 100_000, n).astype(float)
	75	+ "close": close})
	76	+ if expiry is not None:
	77	+ dte = np.array([(expiry - d.date()).days for d in idx], dtype=float)
	78	+ vol, oi_v = _lifecycle(dte, rng)
	79	+ df["volume"] = vol if tf == "1day" else np.round(vol / 400)
	80	+ if oi:
	81	+ df["open_interest"] = oi_v
	82	+ else:
	83	+ df["volume"] = rng.integers(100, 10_000, n).astype(float)
	84	+ if oi:
	85	+ df["open_interest"] = rng.integers(1_000, 100_000, n).astype(float)
46	86	return df
47	87
48	88
		`@@ -69,10 +109,10 @@ def build_lake(root: Path, start: date = date(2023, 1, 2), end: date = date(2025`
69	109	# individual contracts: archive = up to 2024, update = 2025+ (with overlap for 2025 contracts)
70	110	for tf in TIMEFRAMES:
71	111	for r in ROOTS:
	112	+ base = {"ES": 5000.0, "CL": 70.0, "NG": 3.0, "E6": 1.08}[r]
72	113	for yy in (23, 24, 25, 26):
73		− for mc in CYCLE[r][:: (1 if r == "ES" else 3)]:
74		− exp_month = MONTHS[mc]
75		− exp = date(2000 + yy, exp_month, 15)
	114	+ for mc in CYCLE[r]:
	115	+ exp = contract_expiry(r, yy, mc)
76	116	first = exp - timedelta(days=400)
77	117	last = min(exp, end)
78	118	if first > end:
		`@@ -81,12 +121,13 @@ def build_lake(root: Path, start: date = date(2023, 1, 2), end: date = date(2025`
81	121	d = pq / "futures_contracts" / tf / bucket
82	122	d.mkdir(parents=True, exist_ok=True)
83	123	seed += 1
84		− _bars(r, max(first, date(2022, 1, 3)), last, tf, seed, oi=(tf == "1day")).to_parquet(
85		− d / f"{r}_{mc}{yy}_{tf}.parquet", index=False)
86		− if bucket == "update" and yy == 25: # archive also holds the first half of 2025 contracts
	124	+ kw = dict(oi=(tf == "1day"), base=base + (yy * 4 + MONTHS[mc] / 3) * base / 400, expiry=exp, eth=True,
	125	+ days=INTRADAY_DAYS.get(r, 5))
	126	+ _bars(r, max(first, date(2022, 1, 3)), last, tf, seed, **kw).to_parquet(d / f"{r}_{mc}{yy}_{tf}.parquet", index=False)
	127	+ if bucket == "update" and yy == 25: # archive also holds the first part of 2025 contracts
87	128	d2 = pq / "futures_contracts" / tf / "archive"
88	129	d2.mkdir(parents=True, exist_ok=True)
89		− _bars(r, max(first, date(2022, 1, 3)), min(last, date(2024, 12, 31)), tf, seed, oi=(tf == "1day")).to_parquet(
	130	+ _bars(r, max(first, date(2022, 1, 3)), min(last, date(2024, 12, 31)), tf, seed, **kw).to_parquet(
90	131	d2 / f"{r}_{mc}{yy}_{tf}.parquet", index=False)
91	132	# options: one quarter, one ticker
92	133	d = pq / "options" / "2025_q2"
		`@@ -109,6 +150,8 @@ def build_lake(root: Path, start: date = date(2023, 1, 2), end: date = date(2025`
109	150	w.writerow(["ES", "E-mini S&P 500 (CME) ", "2008-01-02", str(end)])
110	151	w.writerow(["CL", "Crude Oil WTI (NYMEX) ", "2008-01-02", str(end)])
111	152	w.writerow(["NG", "Natural Gas (NYMEX) ", "2008-01-02", str(end)])
	153	+ w.writerow(["E6", "Euro FX Futures (CME) ", "2008-01-02", str(end)])
	154	+ w.writerow(["ZK", "Unknown Product (XXX) ", "2008-01-02", str(end)])
112	155	(root / "state").mkdir(exist_ok=True)
113	156
114	157

added tests/test_futures_api.py +384 −0

		`@@ -0,0 +1,384 @@`
	1	+"""Integration tests — backfill + the 7 `/v1/futures/*` endpoints on the synthetic lake."""
	2	+from __future__ import annotations
	3	+
	4	+import io
	5	+import os
	6	+import subprocess
	7	+import sys
	8	+from datetime import date
	9	+from pathlib import Path
	10	+
	11	+import pandas as pd
	12	+import pytest
	13	+
	14	+TODAY = date(2025, 7, 1) # fixture lake ends 2025-06-30
	15	+
	16	+
	17	+@pytest.fixture(scope="module")
	18	+def backfilled(app, lake):
	19	+ from futures.backfill import run_backfill
	20	+ s1 = run_backfill(today=TODAY)
	21	+ s2 = run_backfill(today=TODAY) # idempotent
	22	+ assert s1["contracts"] == s2["contracts"] > 0
	23	+ return s1
	24	+
	25	+
	26	+# ---- backfill ------------------------------------------------------------------------------------------------
	27	+
	28	+def test_backfill_summary_and_tables(backfilled, app):
	29	+ from core.db import session
	30	+ from futures.models import FuturesContract, FuturesRoot
	31	+ from sqlalchemy import select
	32	+ assert backfilled["rule_based"] == backfilled["contracts"] # ES, CL, NG, E6 all have exchange rules
	33	+ assert backfilled["active"] > 0 and backfilled["seconds"] < 30
	34	+ with session() as s:
	35	+ es = s.execute(select(FuturesContract).where(FuturesContract.symbol == "ESH25")).scalar_one()
	36	+ assert es.expiration_date == date(2025, 3, 21) and es.expiration_source == "rule"
	37	+ assert es.first_data_date is not None and es.last_data_date == date(2025, 3, 21)
	38	+ assert es.status == "expired" and es.volume_avg_daily > 0 and es.open_interest_last > 0
	39	+ cl = s.execute(select(FuturesContract).where(FuturesContract.symbol == "CLJ25")).scalar_one()
	40	+ assert cl.first_notice_date == date(2025, 3, 21) and cl.expiration_date == date(2025, 3, 20)
	41	+ root = s.get(FuturesRoot, "ES")
	42	+ assert root.source == "reference" and root.contracts_count >= 14 and root.exchange == "CME"
	43	+ zk = s.get(FuturesRoot, "ZK") # from meta csv only → derived
	44	+ assert zk.source == "derived" and zk.name == "Unknown Product" and zk.exchange == "XXX" and zk.expiry_rule == "data"
	45	+ e6 = s.get(FuturesRoot, "E6")
	46	+ assert '"6E"' in e6.aliases
	47	+
	48	+
	49	+def test_backfill_partial_roots(backfilled, app):
	50	+ from futures.backfill import run_backfill
	51	+ s = run_backfill(roots=["6E"], today=TODAY) # alias accepted through normalize in CLI; here lake root E6 expected
	52	+ assert s["contracts"] == 0 # "6E" is not a lake root: nothing scanned…
	53	+ s = run_backfill(roots=["E6"], today=TODAY)
	54	+ assert s["contracts"] >= 8 and s["roots"] == 1
	55	+
	56	+
	57	+def test_backfill_cli(lake):
	58	+ env = {**os.environ, "HFMD_DATA_ROOT": str(lake), "HFMD_STATE_DB": str(lake / "state" / "hfmd.db"), "HFMD_ENV": "test"}
	59	+ script = Path(__file__).resolve().parents[1] / "scripts" / "backfill_contracts.py"
	60	+ r = subprocess.run([sys.executable, str(script), "--roots", "6E,ES", "--today", str(TODAY), "--json"], env=env,
	61	+ capture_output=True, text=True, timeout=120, check=False)
	62	+ assert r.returncode == 0, r.stderr
	63	+ import json
	64	+ out = json.loads(r.stdout.strip().splitlines()[-1])
	65	+ assert out["contracts"] >= 20 and out["rule_based"] == out["contracts"]
	66	+
	67	+
	68	+# ---- roots / contracts ----------------------------------------------------------------------------------------
	69	+
	70	+def test_roots(client, backfilled):
	71	+ r = client.get("/v1/futures/roots")
	72	+ assert r.status_code == 200
	73	+ body = r.json()
	74	+ roots = {x["root"]: x for x in body["data"]}
	75	+ assert body["meta"]["count"] == len(roots) >= 120
	76	+ assert roots["ES"]["name"] == "E-mini S&P 500" and roots["ES"]["tick_value"] == 12.5 and roots["ES"]["expiry_rule"] == "third_friday"
	77	+ assert roots["ES"]["contracts_count"] >= 14 and roots["ES"]["first_data_date"] and roots["ES"]["source"] == "reference"
	78	+ assert roots["E6"]["aliases"] == ["6E"] and roots["US"]["aliases"] == ["ZB"]
	79	+ assert roots["ZK"]["source"] == "derived" and roots["ZK"]["asset_class"] is None
	80	+ r = client.get("/v1/futures/roots?asset_class=energy&search=crude")
	81	+ assert {x["root"] for x in r.json()["data"]} == {"CL", "MCL", "B"}
	82	+ r = client.get("/v1/futures/roots?exchange=NYMEX")
	83	+ assert {x["root"] for x in r.json()["data"]} >= {"CL", "BZ", "NG", "HO", "RB"}
	84	+ r = client.get("/v1/futures/roots?format=csv")
	85	+ assert r.status_code == 200 and r.text.startswith("root,name,exchange")
	86	+
	87	+
	88	+def test_contracts_list_filters_and_sort(client, backfilled):
	89	+ r = client.get("/v1/futures/ES/contracts")
	90	+ assert r.status_code == 200
	91	+ rows = r.json()["data"]
	92	+ exps = [x["expiration_date"] for x in rows]
	93	+ assert exps == sorted(exps) and len(rows) >= 14
	94	+ assert rows[0]["expiration_source"] == "rule" and rows[0]["timeframes"]["1day"]["rows"] > 0
	95	+ assert rows[0]["files"]["1day"][0].endswith("_1day.parquet")
	96	+ r = client.get("/v1/futures/es/contracts?status=active&sort=-expiration_date")
	97	+ act = r.json()["data"]
	98	+ assert {x["status"] for x in act} == {"active"} and act[0]["expiration_date"] > act[-1]["expiration_date"]
	99	+ assert {x["symbol"] for x in act} == {"ESU25", "ESZ25", "ESH26", "ESM26"}
	100	+ r = client.get("/v1/futures/ES/contracts?from=2024-01-01&to=2024-12-31")
	101	+ assert {x["symbol"] for x in r.json()["data"]} == {"ESH24", "ESM24", "ESU24", "ESZ24"}
	102	+ r = client.get("/v1/futures/6E/contracts")
	103	+ assert r.status_code == 200 and r.json()["meta"]["root"] == "E6" and r.json()["meta"]["count"] >= 8
	104	+ assert client.get("/v1/futures/ES/contracts?sort=bogus").json()["error"]["code"] == "INVALID_PARAMETER"
	105	+ assert client.get("/v1/futures/ES/contracts?status=maybe").json()["error"]["code"] == "INVALID_PARAMETER"
	106	+ r = client.get("/v1/futures/ES/contracts?format=csv")
	107	+ assert r.status_code == 200 and r.headers["X-Row-Count"] == str(len(rows))
	108	+
	109	+
	110	+# ---- bars ------------------------------------------------------------------------------------------------------
	111	+
	112	+def test_contract_bars_daily_json(client, backfilled):
	113	+ r = client.get("/v1/futures/contract/ESH25/bars?interval=1d&from=2025-01-01&to=2025-01-10")
	114	+ assert r.status_code == 200
	115	+ body = r.json()
	116	+ assert body["meta"]["symbol"] == "ESH25" and body["meta"]["interval"] == "1d" and body["meta"]["next_cursor"] is None
	117	+ assert body["meta"]["expiration_date"] == "2025-03-21" and body["meta"]["expiration_source"] == "rule"
	118	+ dates = [x["datetime"] for x in body["data"]]
	119	+ assert dates == sorted(dates) and dates[0] == "2025-01-01" and dates[-1] == "2025-01-10" # fixture = plain weekdays
	120	+ assert set(body["data"][0]) == {"symbol", "datetime", "open", "high", "low", "close", "volume", "open_interest"}
	121	+ assert r.headers["X-Row-Count"] == str(len(dates))
	122	+
	123	+
	124	+def test_contract_bars_intraday_utc_and_sessions(client, backfilled):
	125	+ # November 2024 (EST, UTC-5): RTH starts 09:30 ET = 14:30Z; ETH block 08:00 ET = 13:00Z
	126	+ r = client.get("/v1/futures/contract/es_h25/bars?interval=1m&session=rth&limit=2")
	127	+ d = r.json()["data"]
	128	+ assert d[0]["datetime"].endswith("T14:30:00Z") and d[1]["datetime"].endswith("T14:31:00Z")
	129	+ assert "open_interest" not in d[0]
	130	+ r = client.get("/v1/futures/contract/ESH2025/bars?interval=1m&session=eth&limit=1")
	131	+ assert r.json()["data"][0]["datetime"].endswith("T13:00:00Z")
	132	+ r = client.get("/v1/futures/contract/ES_H25/bars?interval=1m&session=all&limit=1")
	133	+ assert r.json()["data"][0]["datetime"].endswith("T13:00:00Z")
	134	+ # June 2025 (EDT, UTC-4): 09:30 ET = 13:30Z — DST handled
	135	+ r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&session=rth&from=2025-06-16&limit=1")
	136	+ assert r.json()["data"][0]["datetime"] == "2025-06-16T13:30:00Z"
	137	+ # timeframe alias and 1hour interval
	138	+ r = client.get("/v1/futures/contract/ESU25/bars?timeframe=1hour&limit=1&from=2025-06-16")
	139	+ assert r.status_code == 200 and r.json()["meta"]["interval"] == "1h"
	140	+ # FX root uses its own RTH window (08:20 ET → 13:20Z in December)
	141	+ r = client.get("/v1/futures/contract/6EZ25/bars?interval=1m&session=rth&limit=1")
	142	+ assert r.json()["meta"]["symbol"] == "E6Z25" and r.json()["data"][0]["datetime"].endswith("T13:20:00Z")
	143	+
	144	+
	145	+def test_contract_bars_from_to_datetime_bounds(client, backfilled):
	146	+ r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&from=2025-06-16T13:30:00Z&to=2025-06-16T13:35:00Z")
	147	+ d = r.json()["data"]
	148	+ assert [x["datetime"] for x in d] == [f"2025-06-16T13:3{i}:00Z" for i in range(5)]
	149	+ r = client.get("/v1/futures/contract/ESU25/bars?interval=1m&from=not-a-date")
	150	+ assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
	151	+
	152	+
	153	+def test_cursor_pagination(client, backfilled):
	154	+ r1 = client.get("/v1/futures/contract/ESH25/bars?interval=5m&limit=7")
	155	+ b1 = r1.json()
	156	+ assert len(b1["data"]) == 7 and b1["meta"]["next_cursor"]
	157	+ r2 = client.get(f"/v1/futures/contract/ESH25/bars?interval=5m&limit=7&cursor={b1['meta']['next_cursor']}")
	158	+ b2 = r2.json()
	159	+ assert len(b2["data"]) == 7 and b2["data"][0]["datetime"] > b1["data"][-1]["datetime"]
	160	+ full = client.get("/v1/futures/contract/ESH25/bars?interval=5m&limit=14").json()["data"]
	161	+ assert [x["datetime"] for x in b1["data"] + b2["data"]] == [x["datetime"] for x in full]
	162	+ last = client.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=100000").json()
	163	+ assert last["meta"]["next_cursor"] is None
	164	+ assert client.get("/v1/futures/contract/ESH25/bars?cursor=%%%").json()["error"]["code"] == "INVALID_PARAMETER"
	165	+ assert client.get("/v1/futures/contract/ESH25/bars?limit=0").json()["error"]["code"] == "INVALID_PARAMETER"
	166	+
	167	+
	168	+def test_bars_csv_and_parquet(client, backfilled):
	169	+ r = client.get("/v1/futures/contract/ESH25/bars?interval=1m&limit=3&format=csv")
	170	+ assert r.status_code == 200 and r.headers["content-type"].startswith("text/csv") and r.headers["X-Row-Count"] == "3"
	171	+ lines = r.text.strip().splitlines()
	172	+ assert lines[0] == "symbol,datetime,open,high,low,close,volume" and lines[1].split(",")[1].endswith("Z")
	173	+ r = client.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=2&format=csv")
	174	+ assert r.text.splitlines()[1].split(",")[1] == "2024-02-15" # daily = plain date
	175	+ r = client.get("/v1/futures/contract/ESH25/bars?interval=1m&limit=50&format=parquet")
	176	+ assert r.status_code == 200 and r.headers["content-type"] == "application/vnd.apache.parquet" and r.headers["X-Row-Count"] == "50"
	177	+ df = pd.read_parquet(io.BytesIO(r.content))
	178	+ assert len(df) == 50 and str(df["datetime"].dtype).endswith("UTC]") and list(df.columns)[:2] == ["symbol", "datetime"]
	179	+
	180	+
	181	+def test_bars_merge_archive_and_update_dedup(client, backfilled):
	182	+ """ESH25 exists in both buckets (archive ≤ 2024-12-31, update full): no duplicate dates."""
	183	+ d = client.get("/v1/futures/contract/ESH25/bars?interval=1d&limit=100000").json()["data"]
	184	+ dates = [x["datetime"] for x in d]
	185	+ assert len(dates) == len(set(dates)) and dates[0] < "2024-12-31" < dates[-1]
	186	+
	187	+
	188	+# ---- coverage ------------------------------------------------------------------------------------------------
	189	+
	190	+def test_coverage(client, backfilled):
	191	+ r = client.get("/v1/futures/contract/ESH25/coverage")
	192	+ assert r.status_code == 200
	193	+ c = r.json()["data"]
	194	+ assert c["symbol"] == "ESH25" and c["expiration_source"] == "rule" and c["expiration_date"] == "2025-03-21"
	195	+ assert c["intervals"]["1d"]["available"] and c["intervals"]["1d"]["open_interest"] is True and c["intervals"]["1d"]["rows"] > 200
	196	+ assert c["intervals"]["1m"]["available"] and c["intervals"]["1m"]["open_interest"] is False
	197	+ assert set(c["intervals"]["1m"]["files"]) == {"archive", "update"}
	198	+ assert c["gaps"] == [] and isinstance(c["notes"], list)
	199	+
	200	+
	201	+# ---- chain / term structure ------------------------------------------------------------------------------------
	202	+
	203	+def test_chain(client, backfilled):
	204	+ r = client.get("/v1/futures/ES/chain?as_of=2024-09-10")
	205	+ assert r.status_code == 200
	206	+ body = r.json()
	207	+ rows = body["data"]
	208	+ assert body["meta"]["front"] == "ESU24" and rows[0]["position"] == 1 and rows[0]["days_to_expiry"] == 10
	209	+ assert [x["symbol"] for x in rows] == ["ESU24", "ESZ24", "ESH25", "ESM25", "ESU25"]
	210	+ assert rows[0]["close"] and rows[0]["last_date"] == "2024-09-10" and rows[0]["open_interest"] > 0
	211	+ assert all(x["expiration_date"] >= "2024-09-10" for x in rows)
	212	+ r = client.get("/v1/futures/ES/chain?as_of=2010-01-01")
	213	+ assert r.json()["data"] == [] and r.json()["meta"]["note"]
	214	+ assert client.get("/v1/futures/ES/chain?as_of=yesterday").json()["error"]["code"] == "INVALID_PARAMETER"
	215	+ assert client.get("/v1/futures/ES/chain?as_of=2024-09-10&format=csv").status_code == 200
	216	+
	217	+
	218	+def test_term_structure(client, backfilled):
	219	+ r = client.get("/v1/futures/cl/term-structure?as_of=2025-06-30")
	220	+ assert r.status_code == 200
	221	+ body = r.json()
	222	+ rows, meta = body["data"], body["meta"]
	223	+ assert meta["root"] == "CL" and meta["structure"] in ("contango", "backwardation", "flat")
	224	+ assert rows[0]["slope_annualized"] is None and rows[0]["settle"] == meta["front_settle"]
	225	+ assert rows[1]["slope_annualized"] is not None and rows[1]["spread_vs_front"] == pytest.approx(rows[1]["settle"] - rows[0]["settle"])
	226	+ dte = [x["days_to_expiry"] for x in rows]
	227	+ assert dte == sorted(dte) and dte[0] >= 0
	228	+ if meta["structure"] == "contango":
	229	+ assert rows[1]["settle"] > rows[0]["settle"]
	230	+
	231	+
	232	+# ---- continuous ------------------------------------------------------------------------------------------------
	233	+
	234	+def test_continuous_calendar_roll_and_back_adjustment(client, backfilled):
	235	+ r = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=none&from=2024-03-01&to=2024-03-31")
	236	+ assert r.status_code == 200
	237	+ raw = r.json()
	238	+ syms = [x["symbol"] for x in raw["data"]]
	239	+ assert syms[0] == "ESH24" and syms[-1] == "ESM24" and syms == sorted(syms, key=syms.index)
	240	+ assert raw["meta"]["roll_dates"] == [pytest.approx(raw["meta"]["roll_dates"][0])]
	241	+ rd = raw["meta"]["roll_dates"][0]
	242	+ assert rd["date"] == "2024-03-18" and rd["from_symbol"] == "ESH24" and rd["to_symbol"] == "ESM24" and rd["adjusted"]
	243	+ assert rd["gap_session"] == "2024-03-15" and rd["gap"] == pytest.approx(rd["ratio"] * 0 + rd["gap"])
	244	+ # switch day: last ESH24 bar is its expiry (Fri 15), first ESM24 bar is Mon 18
	245	+ last_h = max(x["datetime"] for x in raw["data"] if x["symbol"] == "ESH24")
	246	+ first_m = min(x["datetime"] for x in raw["data"] if x["symbol"] == "ESM24")
	247	+ assert last_h == "2024-03-15" and first_m == "2024-03-18"
	248	+ adj = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=back_adjusted&from=2024-03-01&to=2024-03-31").json()
	249	+ raw_close = {(x["symbol"], x["datetime"]): x["close"] for x in raw["data"]}
	250	+ offsets = {}
	251	+ for x in adj["data"]:
	252	+ offsets.setdefault(x["symbol"], set()).add(round(x["close"] - raw_close[(x["symbol"], x["datetime"])], 6))
	253	+ assert len(offsets["ESH24"]) == 1 and len(offsets["ESM24"]) == 1 # constant shift per segment
	254	+ assert (offsets["ESH24"].pop() - offsets["ESM24"].pop()) == pytest.approx(rd["gap"], abs=1e-6)
	255	+ assert adj["data"][0]["volume"] == raw["data"][0]["volume"] # volume never adjusted
	256	+ # the last segment (latest contract) is unadjusted
	257	+ tail = client.get("/v1/futures/ES/continuous?roll=calendar&adjust=back_adjusted&from=2025-06-25").json()
	258	+ tail_raw = client.get("/v1/futures/contract/ESU25/bars?interval=1d&from=2025-06-25").json()
	259	+ assert tail["meta"]["unadjusted_symbol"] == "ESM26"
	260	+ assert {x["symbol"] for x in tail["data"]} == {"ESU25"} or tail["data"][-1]["symbol"] == tail["meta"]["unadjusted_symbol"]
	261	+ if {x["symbol"] for x in tail["data"]} == {"ESU25"}:
	262	+ assert tail["data"][-1]["close"] != tail_raw["data"][-1]["close"] or tail["meta"]["rolls_total"] > 0
	263	+
	264	+
	265	+def test_continuous_volume_roll_before_expiry_and_ratio(client, backfilled):
	266	+ r = client.get("/v1/futures/ES/continuous?roll=volume&adjust=ratio_adjusted&from=2024-03-01&to=2024-03-31").json()
	267	+ rd = r["meta"]["roll_dates"]
	268	+ assert len(rd) == 1 and rd[0]["from_symbol"] == "ESH24" and rd[0]["to_symbol"] == "ESM24"
	269	+ assert "2024-03-08" < rd[0]["date"] < "2024-03-15" # volume roll a few sessions before expiry
	270	+ assert rd[0]["ratio"] and rd[0]["ratio"] != 1.0
	271	+ assert r["meta"]["roll"] == "volume" and r["meta"]["adjust"] == "ratio_adjusted"
	272	+ oi = client.get("/v1/futures/ES/continuous?roll=open_interest&from=2024-03-01&to=2024-03-31").json()
	273	+ assert oi["meta"]["roll_dates"][0]["to_symbol"] == "ESM24" and oi["meta"]["roll_dates"][0]["date"] < "2024-03-18"
	274	+
	275	+
	276	+def test_continuous_first_notice_roll(client, backfilled):
	277	+ r = client.get("/v1/futures/CL/continuous?roll=first_notice&from=2024-03-15&to=2024-03-25").json()
	278	+ rd = r["meta"]["roll_dates"]
	279	+ assert len(rd) == 1 and rd[0]["date"] == "2024-03-21" and rd[0]["from_symbol"] == "CLJ24" and rd[0]["to_symbol"] == "CLN24"
	280	+
	281	+
	282	+def test_continuous_depth(client, backfilled):
	283	+ r1 = client.get("/v1/futures/ES/continuous?roll=calendar&from=2024-03-01&to=2024-03-31").json()
	284	+ r2 = client.get("/v1/futures/ES/continuous?roll=calendar&depth=2&from=2024-03-01&to=2024-03-31").json()
	285	+ r3 = client.get("/v1/futures/ES/continuous?roll=calendar&depth=3&from=2024-03-01&to=2024-03-31").json()
	286	+ assert [x["symbol"] for x in r2["data"]][0] == "ESM24" and r2["data"][-1]["symbol"] == "ESU24"
	287	+ assert r3["data"][0]["symbol"] == "ESU24" and r3["data"][-1]["symbol"] == "ESZ24"
	288	+ assert len(r1["data"]) == len(r2["data"]) == len(r3["data"])
	289	+ assert r2["meta"]["roll_dates"][0]["from_symbol"] == "ESM24" and r2["meta"]["roll_dates"][0]["to_symbol"] == "ESU24"
	290	+ assert client.get("/v1/futures/ES/continuous?depth=4").json()["error"]["code"] == "INVALID_PARAMETER"
	291	+
	292	+
	293	+def test_continuous_intraday_applies_daily_schedule(client, backfilled):
	294	+ r = client.get("/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100000")
	295	+ assert r.status_code == 200
	296	+ body = r.json()
	297	+ syms = [x["symbol"] for x in body["data"]]
	298	+ assert set(syms) == {"ESM25", "ESU25"} and syms == sorted(syms, key=syms.index)
	299	+ ts = [x["datetime"] for x in body["data"]]
	300	+ assert ts == sorted(ts) and all(t.endswith("Z") for t in ts) and ts[0] >= "2025-06-10T13:30:00Z"
	301	+ assert len(body["meta"]["roll_dates"]) == 1 and body["meta"]["roll_dates"][0]["to_symbol"] == "ESU25"
	302	+ assert "open_interest" not in body["data"][0]
	303	+ page = client.get("/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100").json()
	304	+ nxt = client.get(f"/v1/futures/ES/continuous?interval=1m&roll=volume&from=2025-06-10&to=2025-06-25&session=rth&limit=100&cursor={page['meta']['next_cursor']}").json()
	305	+ assert nxt["data"][0]["datetime"] > page["data"][-1]["datetime"]
	306	+ r = client.get("/v1/futures/ES/continuous?interval=1m&from=2025-06-10&to=2025-06-12&format=parquet")
	307	+ assert r.status_code == 200 and int(r.headers["X-Row-Count"]) > 0
	308	+ r = client.get("/v1/futures/ES/continuous?from=2025-06-10&to=2025-06-12&format=csv")
	309	+ assert r.text.splitlines()[0].startswith("symbol,datetime")
	310	+
	311	+
	312	+def test_continuous_alias_root_and_empty_window(client, backfilled):
	313	+ r = client.get("/v1/futures/6E/continuous?roll=calendar&from=2025-06-20&to=2025-06-30")
	314	+ assert r.status_code == 200 and r.json()["meta"]["root"] == "E6" and r.json()["data"]
	315	+ r = client.get("/v1/futures/ES/continuous?from=2010-01-01&to=2010-01-31")
	316	+ assert r.status_code == 200 and r.json()["data"] == [] and r.json()["meta"]["roll_dates"] == []
	317	+
	318	+
	319	+# ---- errors ------------------------------------------------------------------------------------------------------
	320	+
	321	+@pytest.mark.parametrize("url, status, code", [
	322	+ ("/v1/futures/contract/BOGUS/bars", 400, "INVALID_CONTRACT_SYMBOL"),
	323	+ ("/v1/futures/contract/ESZ5/coverage", 400, "INVALID_CONTRACT_SYMBOL"),
	324	+ ("/v1/futures/contract/ESZ99/bars", 404, "CONTRACT_NOT_FOUND"),
	325	+ ("/v1/futures/contract/ESZ99/coverage", 404, "CONTRACT_NOT_FOUND"),
	326	+ ("/v1/futures/NOPE/chain", 404, "ROOT_NOT_FOUND"),
	327	+ ("/v1/futures/NOPE/contracts", 404, "ROOT_NOT_FOUND"),
	328	+ ("/v1/futures/NOPE/continuous", 404, "ROOT_NOT_FOUND"),
	329	+ ("/v1/futures/NOPE/term-structure", 404, "ROOT_NOT_FOUND"),
	330	+ ("/v1/futures/ES/continuous?roll=weird", 400, "INVALID_PARAMETER"),
	331	+ ("/v1/futures/ES/continuous?adjust=weird", 400, "INVALID_PARAMETER"),
	332	+ ("/v1/futures/contract/ESH25/bars?interval=2h", 400, "INVALID_PARAMETER"),
	333	+ ("/v1/futures/contract/ESH25/bars?session=lunch", 400, "INVALID_PARAMETER"),
	334	+ ("/v1/futures/contract/ESH25/bars?format=xml", 400, "INVALID_PARAMETER"),
	335	+])
	336	+def test_error_envelopes(client, backfilled, url, status, code):
	337	+ r = client.get(url)
	338	+ assert r.status_code == status
	339	+ body = r.json()
	340	+ assert body["error"]["code"] == code and body["error"]["docs"].endswith(f"#{code.lower()}") and body["detail"]
	341	+
	342	+
	343	+def test_tier_row_cap_is_enforced_via_request_state(client, backfilled, app):
	344	+ """clamp_limit(..., request=request) honours request.state.max_rows set by a limiter middleware."""
	345	+ from starlette.middleware.base import BaseHTTPMiddleware
	346	+
	347	+ class Cap(BaseHTTPMiddleware):
	348	+ async def dispatch(self, request, call_next):
	349	+ request.state.max_rows = 10
	350	+ return await call_next(request)
	351	+ # build a throwaway app sharing the routes
	352	+ from core import errors
	353	+ from fastapi import FastAPI
	354	+ from fastapi.testclient import TestClient
	355	+ from futures.routes import router
	356	+ a = FastAPI()
	357	+ errors.install(a)
	358	+ a.add_middleware(Cap)
	359	+ a.include_router(router)
	360	+ with TestClient(a) as c:
	361	+ r = c.get("/v1/futures/contract/ESH25/bars?limit=50")
	362	+ assert r.status_code == 400 and r.json()["error"]["code"] == "ROW_LIMIT_EXCEEDED" and r.json()["error"]["details"]["max_rows"] == 10
	363	+ r = c.get("/v1/futures/contract/ESH25/bars?limit=5")
	364	+ assert r.status_code == 200 and r.headers["X-Row-Count"] == "5"
	365	+
	366	+
	367	+# ---- OpenAPI -----------------------------------------------------------------------------------------------------
	368	+
	369	+def test_openapi_documents_the_seven_endpoints(client, backfilled):
	370	+ spec = client.get("/openapi.json").json()
	371	+ paths = spec["paths"]
	372	+ expected = ["/v1/futures/roots", "/v1/futures/{root}/contracts", "/v1/futures/contract/{symbol}/bars",
	373	+ "/v1/futures/contract/{symbol}/coverage", "/v1/futures/{root}/chain", "/v1/futures/{root}/continuous",
	374	+ "/v1/futures/{root}/term-structure"]
	375	+ for p in expected:
	376	+ op = paths[p]["get"]
	377	+ assert op["tags"] == ["futures"] and op["summary"] and len(op["description"]) > 80
	378	+ assert "200" in op["responses"] and "429" in op["responses"]
	379	+ bars = paths["/v1/futures/contract/{symbol}/bars"]["get"]
	380	+ assert "CONTRACT_NOT_FOUND" in bars["responses"]["404"]["description"]
	381	+ assert "INVALID_CONTRACT_SYMBOL" in bars["responses"]["400"]["description"]
	382	+ assert "example" in bars["responses"]["200"]["content"]["application/json"]
	383	+ names = {p["name"] for p in bars["parameters"]}
	384	+ assert {"interval", "from", "to", "session", "cursor", "limit", "format"} <= names

added tests/test_futures_expiry.py +132 −0

		`@@ -0,0 +1,132 @@`
	1	+"""Unit tests — holiday calendar and every expiry rule against real exchange dates.
	2	+
	3	+Reference dates were cross-checked with the last daily bar of each contract in the production lake
	4	+(FirstRate Data) and the exchange rulebooks (CME Group, ICE, Eurex, Cboe).
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+from datetime import date
	9	+
	10	+import pytest
	11	+from futures import calendar_us as cal
	12	+from futures import expiry
	13	+from futures.specs import SPEC_BY_ROOT, SPECS
	14	+from futures.symbols import parse_symbol
	15	+
	16	+# ---- calendar -------------------------------------------------------------------------------------------
	17	+
	18	+def test_us_holidays_2024():
	19	+ assert sorted(cal.us_holidays(2024)) == [date(2024, 1, 1), date(2024, 1, 15), date(2024, 2, 19), date(2024, 3, 29),
	20	+ date(2024, 5, 27), date(2024, 6, 19), date(2024, 7, 4), date(2024, 9, 2),
	21	+ date(2024, 11, 28), date(2024, 12, 25)]
	22	+
	23	+
	24	+def test_observed_rules():
	25	+ assert date(2021, 12, 24) in cal.us_holidays(2021) # Christmas 2021 on Saturday → Friday
	26	+ assert date(2021, 12, 31) not in cal.us_holidays(2021) # New Year 2022 on Saturday → no observance
	27	+ assert date(2022, 1, 1) not in cal.us_holidays(2022)
	28	+ assert date(2023, 1, 2) in cal.us_holidays(2023) # New Year 2023 on Sunday → Monday
	29	+ assert date(2022, 6, 20) in cal.us_holidays(2022) # Juneteenth 2022 on Sunday → Monday
	30	+ assert date(2021, 6, 18) not in cal.us_holidays(2021) # Juneteenth not observed before 2022
	31	+ assert date(2026, 7, 3) in cal.us_holidays(2026) # July 4 2026 on Saturday → Friday
	32	+
	33	+
	34	+def test_easter_and_eurex():
	35	+ assert cal.easter(2024) == date(2024, 3, 31) and cal.easter(2025) == date(2025, 4, 20) and cal.easter(2030) == date(2030, 4, 21)
	36	+ hs = cal.eurex_holidays(2025)
	37	+ assert {date(2025, 4, 18), date(2025, 4, 21), date(2025, 5, 1), date(2025, 12, 24), date(2025, 12, 26), date(2025, 12, 31)} <= hs
	38	+
	39	+
	40	+def test_business_day_helpers():
	41	+ assert not cal.is_business_day(date(2024, 12, 25))
	42	+ assert not cal.is_business_day(date(2024, 12, 28))
	43	+ assert cal.add_business_days(date(2024, 12, 24), 1) == date(2024, 12, 26)
	44	+ assert cal.add_business_days(date(2024, 12, 31), -7) == date(2024, 12, 19)
	45	+ assert cal.last_business_day_of_month(2024, 11) == date(2024, 11, 29)
	46	+ assert cal.first_business_day_of_month(2025, 1) == date(2025, 1, 2)
	47	+ assert cal.nth_business_day_of_month(2024, 12, 10) == date(2024, 12, 13)
	48	+ assert cal.business_days_between(date(2024, 12, 20), date(2024, 12, 27)) == 3 # 23, 24, 26
	49	+ assert cal.previous_business_day(date(2024, 12, 15)) == date(2024, 12, 13)
	50	+ assert cal.next_business_day(date(2024, 12, 25)) == date(2024, 12, 26)
	51	+ assert cal.shift_month(2024, 1, -1) == (2023, 12) and cal.shift_month(2024, 12, 1) == (2025, 1)
	52	+
	53	+
	54	+# ---- rules vs real dates ----------------------------------------------------------------------------------
	55	+
	56	+REAL = [
	57	+ # symbol, rule, last trading date, justification
	58	+ ("ESZ24", "third_friday", "2024-12-20", "3rd Friday of December 2024"),
	59	+ ("ESM24", "third_friday", "2024-06-21", "3rd Friday of June 2024"),
	60	+ ("FDAXZ24", "third_friday_eurex", "2024-12-20", "3rd Friday (Eurex calendar)"),
	61	+ ("CLZ24", "cl_rule", "2024-11-20", "25 Nov 2024 (Mon) is a business day → 3 business days before = Wed 20 Nov"),
	62	+ ("CLF25", "cl_rule", "2024-12-19", "25 Dec is a holiday → business day before = 24 Dec → 3 business days before = 19 Dec"),
	63	+ ("CLG25", "cl_rule", "2025-01-21", "25 Jan 2025 (Sat) → Fri 24 Jan → 3 business days before (MLK 20 Jan excluded) = Tue 21 Jan"),
	64	+ ("MCLZ24", "cl_minus_1", "2024-11-19", "one business day before CLZ24"),
	65	+ ("NGZ24", "ng_rule", "2024-11-26", "3 business days before 1 Dec 2024 (Thanksgiving 28 Nov excluded)"),
	66	+ ("NGF25", "ng_rule", "2024-12-27", "3 business days before 1 Jan 2025"),
	67	+ ("GCZ24", "metals_rule", "2024-12-27", "3rd last business day of December 2024 (31, 30, 27)"),
	68	+ ("GCG25", "metals_rule", "2025-02-26", "3rd last business day of February 2025"),
	69	+ ("PLF25", "metals_rule", "2025-01-29", "3rd last business day of January 2025"),
	70	+ ("ZNZ24", "treasury_rule", "2024-12-19", "7th business day before the last business day (31 Dec) — 25 Dec excluded"),
	71	+ ("ZNH25", "treasury_rule", "2025-03-20", "7th business day before 31 Mar 2025"),
	72	+ ("USZ24", "treasury_rule", "2024-12-19", "same rule for the 30-year bond (lake root US = ZB)"),
	73	+ ("ZTZ24", "last_business_day", "2024-12-31", "2-year note: last business day of the contract month"),
	74	+ ("ZCZ24", "grains_rule", "2024-12-13", "business day prior to 15 Dec 2024 (Sunday) → Fri 13 Dec"),
	75	+ ("ZSX24", "grains_rule", "2024-11-14", "business day prior to Fri 15 Nov 2024"),
	76	+ ("E6Z24", "fx_rule", "2024-12-16", "2 business days before the 3rd Wednesday (18 Dec 2024)"),
	77	+ ("E6H25", "fx_rule", "2025-03-17", "2 business days before Wed 19 Mar 2025"),
	78	+ ("DXZ24", "fx_rule", "2024-12-16", "ICE dollar index follows the same rule"),
	79	+ ("VXZ24", "vx_rule", "2024-12-18", "Wednesday 30 days before the 3rd Friday of January 2025 (17 Jan)"),
	80	+ ("VXF25", "vx_rule", "2025-01-22", "30 days before Fri 21 Feb 2025"),
	81	+ ("VXH25", "vx_rule", "2025-03-18", "3rd Friday of April 2025 is Good Friday → SPX expires Thu 17 Apr → 30 days before = Tue 18 Mar"),
	82	+ ("BTCZ24", "last_friday", "2024-12-27", "last Friday of December 2024"),
	83	+ ("HOZ24", "prior_month_last_business_day", "2024-11-29", "last business day of November 2024"),
	84	+ ("SBH25", "prior_month_last_business_day", "2025-02-28", "Sugar #11: last business day of the month preceding"),
	85	+ ("BZZ24", "bz_rule", "2024-10-31", "last business day of the 2nd month preceding December"),
	86	+ ("LEZ24", "last_business_day", "2024-12-31", "Live cattle: last business day of the contract month"),
	87	+ ("HEZ24", "he_rule", "2024-12-13", "Lean hogs: 10th business day of December 2024"),
	88	+ ("GFX24", "gf_rule", "2024-11-21", "Feeder cattle: last Thursday is Thanksgiving → Thursday before"),
	89	+ ("GFQ24", "gf_rule", "2024-08-29", "Feeder cattle: last Thursday of August 2024"),
	90	+ ("KCZ24", "kc_rule", "2024-12-18", "Coffee: 8 business days before 31 Dec 2024 (25 Dec excluded)"),
	91	+ ("CCZ24", "cc_rule", "2024-12-13", "Cocoa: 11 business days before 31 Dec 2024"),
	92	+ ("CTZ24", "ct_rule", "2024-12-06", "Cotton: 17 business days from the end of the spot month"),
	93	+ ("OJF25", "oj_rule", "2025-01-10", "FCOJ: 14th business day before 31 Jan 2025 (MLK excluded)"),
	94	+ ("FGBLZ24", "bund_rule", "2024-12-06", "Bund: delivery 10 Dec 2024 → 2 exchange days before"),
	95	+ ("SR3Z24", "sr3_rule", "2025-03-18", "3-month SOFR: business day before the 3rd Wednesday of March 2025"),
	96	+ ("NKDZ24", "second_friday_minus_1", "2024-12-12", "Nikkei: business day before the 2nd Friday (13 Dec 2024)"),
	97	+]
	98	+
	99	+
	100	+@pytest.mark.parametrize("symbol, rule, expected, why", REAL, ids=[r[0] for r in REAL])
	101	+def test_rules_against_real_dates(symbol, rule, expected, why):
	102	+ cs = parse_symbol(symbol)
	103	+ assert expiry.compute(rule, cs.year, cs.month) == date.fromisoformat(expected), why
	104	+
	105	+
	106	+def test_spec_rule_matches_expected_family():
	107	+ """The reference table must point each verified root at the verified rule."""
	108	+ for symbol, rule, _, _ in REAL:
	109	+ root = parse_symbol(symbol).root
	110	+ assert SPEC_BY_ROOT[root].expiry_rule == rule, root
	111	+
	112	+
	113	+def test_first_notice_rules():
	114	+ assert expiry.compute_first_notice("prior_month_last_business_day", 2024, 12) == date(2024, 11, 29) # ZN/GC/ZC Z24
	115	+ assert expiry.compute_first_notice("cl", 2024, 12) == date(2024, 11, 21) # CLZ24: day after LTD
	116	+ assert expiry.compute_first_notice("kc", 2024, 12) == date(2024, 11, 20) # 7 business days before 2 Dec
	117	+ assert expiry.compute_first_notice(None, 2024, 12) is None
	118	+
	119	+
	120	+def test_unknown_rule_returns_none():
	121	+ assert expiry.compute("data", 2024, 12) is None
	122	+ assert expiry.compute(None, 2024, 12) is None
	123	+ assert expiry.compute("nope", 2024, 12) is None
	124	+
	125	+
	126	+def test_every_spec_rule_exists():
	127	+ for s in SPECS:
	128	+ assert s.expiry_rule == "data" or s.expiry_rule in expiry.RULES, s.root
	129	+ assert s.first_notice_rule is None or s.first_notice_rule in expiry.FIRST_NOTICE_RULES, s.root
	130	+ assert s.calendar in ("us", "eurex")
	131	+ assert len(s.rth) == 2
	132	+ assert len({s.root for s in SPECS}) == len(SPECS)

added tests/test_futures_perf.py +46 −0

		`@@ -0,0 +1,46 @@`
	1	+"""Performance smoke check — 10 000 bars of one contract (and of a continuous series) under 300 ms (p95 target)."""
	2	+from __future__ import annotations
	3	+
	4	+import time
	5	+from datetime import date
	6	+
	7	+import pytest
	8	+
	9	+BUDGET_MS = 300
	10	+
	11	+
	12	+@pytest.fixture(scope="module")
	13	+def warm(client, app):
	14	+ from futures.backfill import run_backfill
	15	+ run_backfill(today=date(2025, 7, 1))
	16	+ client.get("/v1/futures/contract/ESU25/bars?interval=1m&limit=10") # warm DuckDB / caches
	17	+ client.get("/v1/futures/ES/continuous?interval=1m&from=2025-05-01&limit=10")
	18	+ return True
	19	+
	20	+
	21	+def _timed(client, url: str, runs: int = 5) -> float:
	22	+ best = float("inf")
	23	+ for _ in range(runs):
	24	+ t = time.perf_counter()
	25	+ r = client.get(url)
	26	+ best = min(best, (time.perf_counter() - t) * 1000)
	27	+ assert r.status_code == 200 and r.headers["X-Row-Count"] == "10000"
	28	+ return best
	29	+
	30	+
	31	+@pytest.mark.slow
	32	+def test_contract_10k_bars_under_budget(client, warm):
	33	+ ms = _timed(client, "/v1/futures/contract/ESU25/bars?interval=1m&limit=10000")
	34	+ assert ms < BUDGET_MS, f"10 000 contract bars took {ms:.0f} ms"
	35	+
	36	+
	37	+@pytest.mark.slow
	38	+def test_continuous_10k_bars_under_budget(client, warm):
	39	+ ms = _timed(client, "/v1/futures/ES/continuous?interval=1m&from=2025-05-01&limit=10000")
	40	+ assert ms < BUDGET_MS, f"10 000 continuous bars took {ms:.0f} ms"
	41	+
	42	+
	43	+@pytest.mark.slow
	44	+def test_parquet_10k_under_budget(client, warm):
	45	+ ms = _timed(client, "/v1/futures/contract/ESU25/bars?interval=1m&limit=10000&format=parquet")
	46	+ assert ms < BUDGET_MS, f"10 000 parquet bars took {ms:.0f} ms"

added tests/test_futures_rolls.py +159 −0

		`@@ -0,0 +1,159 @@`
	1	+"""Unit tests — roll schedules, depth, gap computation and back/ratio adjustment on synthetic series."""
	2	+from __future__ import annotations
	3	+
	4	+from datetime import date
	5	+
	6	+import numpy as np
	7	+import pandas as pd
	8	+import pytest
	9	+from futures.rolls import (
	10	+ Segment,
	11	+ adjustment_offsets,
	12	+ apply_adjustment,
	13	+ depth_schedule,
	14	+ roll_gaps,
	15	+ roll_schedule,
	16	+ stitch_daily,
	17	+)
	18	+
	19	+A, B, C = "XXH24", "XXM24", "XXU24"
	20	+CONTRACTS = [
	21	+ {"symbol": A, "expiration_date": date(2024, 3, 15), "first_notice_date": date(2024, 3, 1), "first_data_date": date(2024, 1, 2), "last_data_date": date(2024, 3, 15)},
	22	+ {"symbol": B, "expiration_date": date(2024, 6, 21), "first_notice_date": date(2024, 6, 3), "first_data_date": date(2024, 1, 2), "last_data_date": date(2024, 6, 21)},
	23	+ {"symbol": C, "expiration_date": date(2024, 9, 20), "first_notice_date": None, "first_data_date": date(2024, 1, 2), "last_data_date": date(2024, 6, 30)},
	24	+]
	25	+
	26	+
	27	+def _daily() -> pd.DataFrame:
	28	+ """Sessions 2024-01-02 → 2024-06-28; A: close 100 flat, B: 105, C: 110. Volume: A dominates until 03-08,
	29	+ B > A on 03-11 and 03-12 (2 consecutive) → volume roll on 03-13. OI: B > A on 03-06/03-07 → roll 03-08."""
	30	+ sessions = pd.bdate_range("2024-01-02", "2024-06-28")
	31	+ rows = []
	32	+ for d in sessions:
	33	+ dd = d.date()
	34	+ if dd <= date(2024, 3, 15):
	35	+ vol_a = 1000 if dd < date(2024, 3, 11) else 100
	36	+ oi_a = 5000 if dd < date(2024, 3, 6) else 50
	37	+ rows.append({"symbol": A, "date": d, "open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": vol_a, "open_interest": oi_a})
	38	+ if dd <= date(2024, 6, 21):
	39	+ rows.append({"symbol": B, "date": d, "open": 105.0, "high": 106.0, "low": 104.0, "close": 105.0, "volume": 500, "open_interest": 2000})
	40	+ rows.append({"symbol": C, "date": d, "open": 110.0, "high": 111.0, "low": 109.0, "close": 110.0, "volume": 10, "open_interest": 100})
	41	+ return pd.DataFrame(rows)
	42	+
	43	+
	44	+def test_calendar_roll():
	45	+ segs = roll_schedule(CONTRACTS, _daily(), "calendar")
	46	+ assert [s.symbol for s in segs] == [A, B, C]
	47	+ assert segs[0].start == date(2024, 1, 2) and segs[0].end == date(2024, 3, 17) # held through expiry (Fri 15), roll Mon 18
	48	+ assert segs[1].start == date(2024, 3, 18) and segs[1].end == date(2024, 6, 23)
	49	+ assert segs[2].start == date(2024, 6, 24) and segs[2].end is None
	50	+
	51	+
	52	+def test_first_notice_roll_falls_back_to_calendar_when_null():
	53	+ segs = roll_schedule(CONTRACTS, _daily(), "first_notice")
	54	+ assert segs[0].end == date(2024, 2, 29) and segs[1].start == date(2024, 3, 1) # roll ON the FND
	55	+ assert segs[1].end == date(2024, 6, 2) and segs[2].start == date(2024, 6, 3)
	56	+
	57	+
	58	+def test_volume_and_oi_rolls():
	59	+ segs = roll_schedule(CONTRACTS, _daily(), "volume")
	60	+ assert segs[0].end == date(2024, 3, 12) and segs[1].start == date(2024, 3, 13)
	61	+ segs = roll_schedule(CONTRACTS, _daily(), "open_interest")
	62	+ assert segs[0].end == date(2024, 3, 7) and segs[1].start == date(2024, 3, 8)
	63	+ # B → C: C never beats B on volume before B's expiry → calendar fallback
	64	+ segs = roll_schedule(CONTRACTS, _daily(), "volume")
	65	+ assert segs[2].start == date(2024, 6, 24)
	66	+
	67	+
	68	+def test_volume_roll_ignores_noise_far_from_expiry():
	69	+ daily = _daily()
	70	+ # a single noisy session in January where B > A must not roll (needs 2 consecutive AND inside the roll window)
	71	+ daily.loc[(daily.symbol == A) & (daily.date == "2024-01-10"), "volume"] = 1
	72	+ segs = roll_schedule(CONTRACTS, daily, "volume")
	73	+ assert segs[1].start == date(2024, 3, 13)
	74	+
	75	+
	76	+def test_skips_contracts_expired_before_previous_roll():
	77	+ contracts = CONTRACTS + [{"symbol": "XXG24", "expiration_date": date(2024, 2, 16), "first_notice_date": None,
	78	+ "first_data_date": date(2024, 1, 2), "last_data_date": date(2024, 2, 16)}]
	79	+ segs = roll_schedule(contracts, _daily(), "calendar")
	80	+ assert [s.symbol for s in segs] == ["XXG24", A, B, C]
	81	+ contracts = CONTRACTS + [{"symbol": "XXZ99", "expiration_date": None, "first_notice_date": None, "first_data_date": None, "last_data_date": None}]
	82	+ assert [s.symbol for s in roll_schedule(contracts, _daily(), "calendar")] == [A, B, C]
	83	+
	84	+
	85	+def test_depth_schedule():
	86	+ front = roll_schedule(CONTRACTS, _daily(), "calendar")
	87	+ d2 = depth_schedule(front, CONTRACTS, 2)
	88	+ assert [(s.symbol, s.start) for s in d2] == [(B, date(2024, 1, 2)), (C, date(2024, 3, 18))]
	89	+ d3 = depth_schedule(front, CONTRACTS, 3)
	90	+ assert [(s.symbol, s.start, s.end) for s in d3] == [(C, date(2024, 1, 2), date(2024, 3, 17))]
	91	+ assert depth_schedule(front, CONTRACTS, 1) is front
	92	+
	93	+
	94	+def test_roll_gaps_and_offsets():
	95	+ daily = _daily()
	96	+ segs = roll_schedule(CONTRACTS, daily, "calendar")
	97	+ rolls = roll_gaps(segs, daily)
	98	+ assert len(rolls) == 2
	99	+ assert rolls[0] == {"date": "2024-03-18", "from_symbol": A, "to_symbol": B, "gap": 5.0, "ratio": 1.05, "gap_session": "2024-03-15", "adjusted": True}
	100	+ assert rolls[1]["gap"] == 5.0 and rolls[1]["from_symbol"] == B and rolls[1]["to_symbol"] == C
	101	+ add, mul = adjustment_offsets(rolls, "back_adjusted")
	102	+ assert add == [10.0, 5.0, 0.0] and mul == [1.0, 1.0, 1.0]
	103	+ add, mul = adjustment_offsets(rolls, "ratio_adjusted")
	104	+ assert add == [0.0, 0.0, 0.0]
	105	+ assert mul[2] == 1.0 and mul[1] == pytest.approx(110 / 105) and mul[0] == pytest.approx(1.05 * 110 / 105)
	106	+ add, mul = adjustment_offsets(rolls, "none")
	107	+ assert add == [0.0, 0.0, 0.0] and mul == [1.0, 1.0, 1.0]
	108	+
	109	+
	110	+def test_stitch_back_adjusted_makes_series_continuous():
	111	+ daily = _daily()
	112	+ segs = roll_schedule(CONTRACTS, daily, "calendar")
	113	+ df, rolls = stitch_daily(segs, daily, "back_adjusted")
	114	+ assert len(rolls) == 2
	115	+ assert df["datetime"].is_monotonic_increasing and df["datetime"].is_unique
	116	+ # latest contract unadjusted, older ones shifted so closes are all 110 → no jump at the rolls
	117	+ assert (df["close"].round(9) == 110.0).all()
	118	+ assert df.loc[df.symbol == A, "volume"].iloc[0] == 1000 # volume untouched
	119	+ df_ratio, _ = stitch_daily(segs, daily, "ratio_adjusted")
	120	+ assert np.allclose(df_ratio["close"], 110.0)
	121	+ df_none, _ = stitch_daily(segs, daily, "none")
	122	+ assert set(df_none["close"].round(6)) == {100.0, 105.0, 110.0}
	123	+ # additive adjustment also shifts open/high/low by the same offset
	124	+ a_rows = df[df.symbol == A].iloc[0]
	125	+ assert a_rows["high"] == pytest.approx(111.0) and a_rows["low"] == pytest.approx(109.0)
	126	+
	127	+
	128	+def test_gap_null_when_no_common_session():
	129	+ daily = _daily()
	130	+ daily = daily[~((daily.symbol == B) & (daily.date < "2024-03-20"))] # B has no bars during A's life
	131	+ segs = roll_schedule(CONTRACTS, daily, "calendar")
	132	+ rolls = roll_gaps(segs, daily)
	133	+ assert rolls[0]["gap"] is None and rolls[0]["adjusted"] is False
	134	+ add, _ = adjustment_offsets(rolls, "back_adjusted")
	135	+ assert add[0] == add[1] == 5.0 # only the B→C gap is applied
	136	+
	137	+
	138	+def test_apply_adjustment_handles_empty_frames():
	139	+ df = apply_adjustment([None, pd.DataFrame(columns=["symbol", "datetime", "open", "high", "low", "close", "volume"])], [0, 0], [1, 1])
	140	+ assert df.empty
	141	+ seg = Segment("X", date(2024, 1, 1), None, 0)
	142	+ assert seg.end is None
	143	+
	144	+
	145	+def test_backfill_gap_detection_and_status(app):
	146	+ from futures.backfill import ( # needs the test env (SQLite path) set by the app fixture
	147	+ _gaps,
	148	+ _status,
	149	+ )
	150	+ days = [date(2024, 12, 20), date(2024, 12, 23), date(2025, 1, 6), date(2025, 1, 7)]
	151	+ gaps = _gaps(days, "us")
	152	+ assert gaps == [(date(2024, 12, 24), date(2025, 1, 5), 7)] # 24, 26, 27, 30, 31 Dec, 2, 3 Jan
	153	+ assert _gaps([date(2024, 12, 20), date(2024, 12, 27)], "us") == [] # 23, 24, 26 = 3 missing → not a gap
	154	+ assert _gaps([], "us") == [] and _gaps([date(2024, 1, 2)], "us") == []
	155	+ today = date(2025, 7, 1)
	156	+ assert _status(date(2025, 6, 30), date(2025, 9, 19), 2025, 9, today) == "active"
	157	+ assert _status(date(2025, 3, 21), date(2025, 3, 21), 2025, 3, today) == "expired"
	158	+ assert _status(date(2025, 6, 28), date(2025, 6, 20), 2025, 6, today) == "active" # recent data → still active
	159	+ assert _status(None, None, 2025, 6, today) == "expired"

added tests/test_futures_symbols.py +82 −0

		`@@ -0,0 +1,82 @@`
	1	+"""Unit tests — contract symbol parsing and root aliases."""
	2	+from __future__ import annotations
	3	+
	4	+import pytest
	5	+from core.errors import ApiError
	6	+from futures.symbols import (
	7	+ ALIASES_OF_ROOT,
	8	+ ROOT_ALIASES,
	9	+ canonical_symbol,
	10	+ expand_year,
	11	+ normalize_root,
	12	+ parse_symbol,
	13	+ symbol_from_file_stem,
	14	+)
	15	+
	16	+
	17	+@pytest.mark.parametrize("raw, root, mc, year", [
	18	+ ("ESZ25", "ES", "Z", 2025),
	19	+ ("ESZ2025", "ES", "Z", 2025),
	20	+ ("ES_Z25", "ES", "Z", 2025),
	21	+ ("ES-Z25", "ES", "Z", 2025),
	22	+ ("esz25", "ES", "Z", 2025),
	23	+ (" esz2025 ", "ES", "Z", 2025),
	24	+ ("CLF26", "CL", "F", 2026),
	25	+ ("M2KH25", "M2K", "H", 2025),
	26	+ ("FDAXZ2024", "FDAX", "Z", 2024),
	27	+ ("SR3Z24", "SR3", "Z", 2024),
	28	+ ("ZKZ25", "ZK", "Z", 2025), # root ending with a month letter
	29	+ ("HHZ25", "HH", "Z", 2025),
	30	+ ("CZ25", "C", "Z", 2025), # 1-char root (London cocoa)
	31	+ ("USZ24", "US", "Z", 2024),
	32	+ ("ESZ99", "ES", "Z", 1999),
	33	+ ("ESZ79", "ES", "Z", 2079),
	34	+ ("ESZ80", "ES", "Z", 1980),
	35	+])
	36	+def test_parse_symbol(raw, root, mc, year):
	37	+ cs = parse_symbol(raw)
	38	+ assert (cs.root, cs.month_code, cs.year) == (root, mc, year)
	39	+ assert cs.short == f"{root}{mc}{year % 100:02d}"
	40	+ assert cs.long == f"{root}{mc}{year}"
	41	+ assert cs.file_stem == f"{root}_{mc}{year % 100:02d}"
	42	+
	43	+
	44	+@pytest.mark.parametrize("raw, root", [
	45	+ ("6EZ25", "E6"), ("6JZ25", "J1"), ("6BZ25", "B6"), ("6AZ25", "A6"), ("6CZ25", "AD"), ("6SZ25", "E1"),
	46	+ ("6NZ25", "N6"), ("6MZ25", "MP"), ("6LZ25", "BR"), ("6ZZ25", "T6"), ("ZBZ25", "US"), ("E6Z25", "E6"),
	47	+])
	48	+def test_aliases_resolve_to_lake_roots(raw, root):
	49	+ assert parse_symbol(raw).root == root
	50	+ assert canonical_symbol(raw) == f"{root}Z25"
	51	+
	52	+
	53	+def test_alias_tables_are_consistent():
	54	+ for alias, root in ROOT_ALIASES.items():
	55	+ assert alias in ALIASES_OF_ROOT[root]
	56	+ assert normalize_root(alias) == root
	57	+ assert normalize_root(alias.lower()) == root
	58	+ assert normalize_root("es") == "ES"
	59	+ assert "6E" in ALIASES_OF_ROOT["E6"] and "ZB" in ALIASES_OF_ROOT["US"]
	60	+
	61	+
	62	+@pytest.mark.parametrize("bad", ["", "ES", "ESA25", "ESZ5", "ESZ2025X", "TOOLONGZ25", "ES Z 25", "Z25", "ES25", "ESZ125",
	63	+ "ES_Z_25", "ÉSZ25", "BOGUS", "ESZ1800"])
	64	+def test_invalid_symbols_raise_400(bad):
	65	+ with pytest.raises(ApiError) as e:
	66	+ parse_symbol(bad)
	67	+ assert e.value.status == 400 and e.value.code == "INVALID_CONTRACT_SYMBOL"
	68	+ assert e.value.payload()["error"]["docs"].endswith("#invalid_contract_symbol")
	69	+
	70	+
	71	+def test_expand_year():
	72	+ assert expand_year(0) == 2000 and expand_year(79) == 2079 and expand_year(80) == 1980 and expand_year(99) == 1999
	73	+ assert expand_year(2031) == 2031
	74	+
	75	+
	76	+def test_symbol_from_file_stem():
	77	+ cs = symbol_from_file_stem("ES_Z24_1day")
	78	+ assert cs is not None and cs.short == "ESZ24" and cs.year == 2024
	79	+ assert symbol_from_file_stem("ES_Z24").short == "ESZ24"
	80	+ assert symbol_from_file_stem("ES_1day") is None
	81	+ assert symbol_from_file_stem("AAPL") is None
	82	+ assert symbol_from_file_stem("ES_ZZ4_1day") is None
	83