SPB Git
spboucher.ai

Repository Public

spbhfmarketdata

Commit

api: v1 robuste — 400 au lieu de 500, tickers partiels, limites de palier, gzip, X-Request-ID, /health étendu, /swagger

- bars/bars_multi/snapshot/options : dates parsées en Python et liées typées (start=bad → 400 INVALID_PARAMETER,
  end < start → 400), > 50 tickers → 400 TOO_MANY_TICKERS, ticker absent dans une requête multi → 200 + en-tête
  X-Missing-Tickers (404 seulement si aucun n'existe), `limit` borné par request.state.max_rows et les plafonds existants,
  LIMIT poussé dans la requête (plus de matérialisation intégrale puis .limit())
- bars_multi/snapshot via core.duck.run_topn (un seul scan filtré) ; suppression des doublons _tls/_cache de main.py
  au profit de core.duck.con()/cached()
- GZipMiddleware(minimum_size=1024) ; core.http.install (X-Request-ID, access log JSON, en-têtes de sécurité) ;
  Cache-Control: public, max-age=86400 sur les fenêtres de barres closes avant aujourd'hui
- /openapi.json : route dédiée (ETag + Cache-Control, 304) au lieu du middleware BaseHTTP ; Swagger déplacé sur
  /swagger, ReDoc sur /redoc — /docs revient au site React ; fallback SPA via core.spa (404 pour chemins inconnus)
- /health : {status, checks, version} 200/503, hors quota ; /v1/status hors quota lignes
- lifespan branché (garde secrets prod, pool de threads borné) ; OpenAPI : X-Request-ID, X-Missing-Tickers, TOO_MANY_TICKERS

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed · Sep 6, 2026 parent b2e3721
4 changed files +685 −204 Browse files
4 changed files
Unified Split
modified hfmarketdata/api/main.py +255 −203
@@ -24,23 +24,37 @@ Contact : contact@spboucher.ai
24 24
25 from __future__ import annotations 25 from __future__ import annotations
26 26
27+import hashlib
27 import os 28 import os
28−import threading 29+from datetime import datetime
29−import time
30 from pathlib import Path 30 from pathlib import Path
31+from zoneinfo import ZoneInfo
31 32
32−import duckdb 33+import pandas as pd
33−from fastapi import FastAPI, HTTPException, Query
34−from fastapi.middleware.cors import CORSMiddleware
35−from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, Response
36−from fastapi.staticfiles import StaticFiles
37−
38 from core import errors as _errors 34 from core import errors as _errors
39−from core.config import settings as _settings 35+from core import health as _health
36+from core import http as _http
37+from core import spa as _spa
38+from core.duck import cached, con, run_topn
39+from core.errors import ApiError
40+from core.lifespan import lifespan
41+from core.params import (
42+ MAX_TICKERS,
43+ bound_limit,
44+ parse_date,
45+ parse_datetime,
46+ parse_range,
47+ split_tickers,
48+)
49+from fastapi import FastAPI, Query, Request
50+from fastapi.middleware.cors import CORSMiddleware
51+from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
52+from fastapi.responses import JSONResponse, PlainTextResponse, Response
53+from starlette.middleware.gzip import GZipMiddleware
40 54
41 __author__ = "Simon-Pierre Boucher" 55 __author__ = "Simon-Pierre Boucher"
42 __contact__ = "contact@spboucher.ai" 56 __contact__ = "contact@spboucher.ai"
43−__version__ = "2.0.0" 57+__version__ = "2.1.0"
44 58
45 DATA_ROOT = Path(os.environ.get("HFMD_DATA_ROOT", "/Volumes/ssd/firstratedata")) 59 DATA_ROOT = Path(os.environ.get("HFMD_DATA_ROOT", "/Volumes/ssd/firstratedata"))
46 PARQUET = DATA_ROOT / "parquet" 60 PARQUET = DATA_ROOT / "parquet"
@@ -56,6 +70,18 @@ DEFAULT_ADJUSTMENT = {
56 } 70 }
57 MAX_LIMIT_JSON = 50_000 71 MAX_LIMIT_JSON = 50_000
58 MAX_LIMIT_CSV = 2_000_000 72 MAX_LIMIT_CSV = 2_000_000
73+ET = ZoneInfo("America/New_York")
74+HISTORY_CACHE = "public, max-age=86400" # bars whose window ends before today never change
75+
76+_LIMIT_DOC = ("Max rows. Silently capped at 50 000 (JSON) / 2 000 000 (CSV) and at your tier's "
77+ "`max_rows_per_request` (see /v1/limits).")
78+_RANGE_DOC = ("ISO 8601 date or datetime (`2024-06-03`, `2024-06-03 10:35:00`, `2024-06-03T10:35:00Z`). "
79+ "Lake timestamps are US/Eastern (intraday) and naive; a bare date means midnight. "
80+ "Malformed values → 400 INVALID_PARAMETER.")
81+_MISSING_DOC = ("Tickers that exist are returned even when others are unknown: the response is 200 with the "
82+ "header `X-Missing-Tickers: A,B` listing the ones not found; 404 only when none exists.")
83+_MISSING_HEADER = {"X-Missing-Tickers": {"schema": {"type": "string"},
84+ "description": "Comma-separated requested tickers that do not exist for this asset / timeframe / adjustment (only when some are missing)."}}
59 85
60 app = FastAPI( 86 app = FastAPI(
61 title="HF Market Data API", 87 title="HF Market Data API",
@@ -68,48 +94,27 @@ app = FastAPI(
68 version=__version__, 94 version=__version__,
69 contact={"name": __author__, "email": __contact__, 95 contact={"name": __author__, "email": __contact__,
70 "url": "https://www.hfmarketdata.io"}, 96 "url": "https://www.hfmarketdata.io"},
97+ # `/docs` belongs to the React documentation site; Swagger UI lives at /swagger, ReDoc at /redoc.
98+ # The OpenAPI document is served by a dedicated route below (ETag + Cache-Control).
99+ openapi_url=None, docs_url=None, redoc_url=None,
100+ lifespan=lifespan,
71 ) 101 )
72 app.add_middleware( 102 app.add_middleware(
73 CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], 103 CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
74− expose_headers=["X-Row-Count", "X-RateLimit-Limit-Requests", "X-RateLimit-Remaining-Requests", 104+ expose_headers=["X-Row-Count", "X-Request-ID", "X-Missing-Tickers", "ETag",
105+ "X-RateLimit-Limit-Requests", "X-RateLimit-Remaining-Requests",
75 "X-RateLimit-Limit-Rows", "X-RateLimit-Remaining-Rows", "X-RateLimit-Reset", "Retry-After"], 106 "X-RateLimit-Limit-Rows", "X-RateLimit-Remaining-Rows", "X-RateLimit-Reset", "Retry-After"],
76 ) 107 )
108+app.add_middleware(GZipMiddleware, minimum_size=1024)
77 _errors.install(app) # uniform {"error": {code, message, docs}} envelope (legacy "detail" kept) 109 _errors.install(app) # uniform {"error": {code, message, docs}} envelope (legacy "detail" kept)
78 110
79−# ------------------------------------------------------------------------------
80−# DuckDB — one connection per thread, read-only usage
81−# ------------------------------------------------------------------------------
82−
83−_tls = threading.local()
84−
85−
86−def db() -> duckdb.DuckDBPyConnection:
87− if not hasattr(_tls, "con"):
88− _tls.con = duckdb.connect()
89− _tls.con.execute("SET threads TO 4")
90− _tls.con.execute("SET enable_object_cache=true")
91− return _tls.con
92−
93 111
94 # ------------------------------------------------------------------------------ 112 # ------------------------------------------------------------------------------
95−# File resolution with a small TTL cache (directory scans are the slow part) 113+# DuckDB: `core.duck.con()` = thread-local cursor on the single per-process database;
114+# `core.duck.cached()` = TTL/LRU cache for directory scans.
96 # ------------------------------------------------------------------------------ 115 # ------------------------------------------------------------------------------
97 116
98−_cache: dict[str, tuple[float, object]] = {} 117+db = con # legacy alias
99−_cache_lock = threading.Lock()
100−CACHE_TTL = 300 # seconds
101−
102−
103−def cached(key: str, builder):
104− now = time.time()
105− with _cache_lock:
106− hit = _cache.get(key)
107− if hit and now - hit[0] < CACHE_TTL:
108− return hit[1]
109− value = builder()
110− with _cache_lock:
111− _cache[key] = (now, value)
112− return value
113 118
114 119
115 def bar_dir(asset: str, timeframe: str, adjustment: str) -> Path: 120 def bar_dir(asset: str, timeframe: str, adjustment: str) -> Path:
@@ -165,38 +170,61 @@ def options_files(ticker: str) -> list[str]:
165 170
166 def _validate(asset: str, timeframe: str, adjustment: str | None) -> str: 171 def _validate(asset: str, timeframe: str, adjustment: str | None) -> str:
167 if asset not in BAR_TYPES: 172 if asset not in BAR_TYPES:
168− raise HTTPException(404, f"Unknown asset type '{asset}'. " 173+ raise ApiError(404, "ASSET_NOT_FOUND", f"Unknown asset type '{asset}'. "
169− f"Available: {', '.join(BAR_TYPES)}") 174+ f"Available: {', '.join(BAR_TYPES)}")
170 if timeframe not in TIMEFRAMES: 175 if timeframe not in TIMEFRAMES:
171− raise HTTPException(400, f"Unknown timeframe '{timeframe}'. " 176+ raise ApiError(400, "INVALID_PARAMETER", f"Unknown timeframe '{timeframe}'. "
172− f"Available: {', '.join(TIMEFRAMES)}") 177+ f"Available: {', '.join(TIMEFRAMES)}")
173 adj = adjustment or DEFAULT_ADJUSTMENT[asset] 178 adj = adjustment or DEFAULT_ADJUSTMENT[asset]
174 available = list_adjustments(asset, timeframe) 179 available = list_adjustments(asset, timeframe)
175 if available and adj not in available: 180 if available and adj not in available:
176− raise HTTPException(400, f"Adjustment '{adj}' not available for " 181+ raise ApiError(400, "INVALID_PARAMETER", f"Adjustment '{adj}' not available for "
177− f"{asset}/{timeframe}. Available: {', '.join(available)}") 182+ f"{asset}/{timeframe}. Available: {', '.join(available)}")
178 return adj 183 return adj
179 184
180 185
181−def _rows_to_response(rel, limit: int, fmt: str, order_col: str): 186+def _resolve_tickers(idx: dict[str, str], names: list[str], where: str) -> tuple[list[str], list[str], dict[str, str]]:
187+ """(found, missing, extra headers): 404 only when NO requested ticker exists."""
188+ found = [t for t in names if t in idx]
189+ missing = [t for t in names if t not in idx]
190+ if not found:
191+ raise ApiError(404, "TICKER_NOT_FOUND", f"Not found in {where}: {', '.join(missing)}",
192+ details={"missing": missing})
193+ return found, missing, ({"X-Missing-Tickers": ",".join(missing)} if missing else {})
194+
195+
196+def _range_conditions(lo: datetime | None, hi: datetime | None) -> tuple[str, list]:
197+ conds, params = [], []
198+ if lo is not None:
199+ conds.append("datetime >= ?"); params.append(lo)
200+ if hi is not None:
201+ conds.append("datetime <= ?"); params.append(hi)
202+ return " AND ".join(conds), params
203+
204+
205+def _history_headers(upper: datetime | None) -> dict[str, str]:
206+ """Immutable history: a window that ends strictly before today (Eastern) can be cached for a day."""
207+ if upper is not None and upper.date() < datetime.now(ET).date():
208+ return {"Cache-Control": HISTORY_CACHE}
209+ return {}
210+
211+
212+def _frame_response(df: pd.DataFrame, fmt: str, headers: dict[str, str] | None = None) -> Response:
213+ """v1 envelope: CSV text or `{"count", "data"}` JSON, always with `X-Row-Count`."""
214+ hdrs = {"X-Row-Count": str(len(df)), **(headers or {})}
182 if fmt == "csv": 215 if fmt == "csv":
183− limit = min(limit, MAX_LIMIT_CSV) 216+ return PlainTextResponse(df.to_csv(index=False), media_type="text/csv", headers=hdrs)
184− rel = rel.limit(limit)
185− df = rel.df()
186− return PlainTextResponse(df.to_csv(index=False), media_type="text/csv",
187− headers={"X-Row-Count": str(len(df))})
188− limit = min(limit, MAX_LIMIT_JSON)
189− rel = rel.limit(limit)
190− df = rel.df()
191 for col in df.columns: 217 for col in df.columns:
192 if str(df[col].dtype).startswith("datetime"): 218 if str(df[col].dtype).startswith("datetime"):
193 df[col] = df[col].astype(str) 219 df[col] = df[col].astype(str)
194− # NaN / ±inf (Greeks manquants dans les vieux trimestres) ne sont pas 220+ # NaN / ±inf (missing Greeks in old quarters) are not strict JSON → null
195− # sérialisables en JSON strict → null
196 df = df.replace([float("inf"), float("-inf")], None) 221 df = df.replace([float("inf"), float("-inf")], None)
197 df = df.astype(object).where(df.notna(), None) 222 df = df.astype(object).where(df.notna(), None)
198− return JSONResponse({"count": len(df), "data": df.to_dict(orient="records")}, 223+ return JSONResponse({"count": len(df), "data": df.to_dict(orient="records")}, headers=hdrs)
199− headers={"X-Row-Count": str(len(df))}) 224+
225+
226+def _hard_max(fmt: str) -> int:
227+ return MAX_LIMIT_CSV if fmt == "csv" else MAX_LIMIT_JSON
200 228
201 229
202 # ------------------------------------------------------------------------------ 230 # ------------------------------------------------------------------------------
@@ -204,16 +232,27 @@ def _rows_to_response(rel, limit: int, fmt: str, order_col: str):
204 # ------------------------------------------------------------------------------ 232 # ------------------------------------------------------------------------------
205 233
206 234
207−@app.get("/health", tags=["meta"]) 235+@app.get("/health", tags=["meta"], summary="Service health",
208−def health(): 236+ description="Liveness + dependency checks: Parquet lake, SQLite state DB, Redis (300 ms timeout) and a witness "
209− return {"status": "ok", "service": "hfmarketdata-api", "version": __version__, 237+ "`read_parquet` (cached 60 s). `status` is `ok` (HTTP 200) or `degraded` (HTTP 503). Never counted "
210− "data_root_present": PARQUET.is_dir()} 238+ "against any quota.",
239+ responses={200: {"description": "Healthy", "content": {"application/json": {"example": {
240+ "status": "ok", "service": "hfmarketdata-api", "version": __version__, "data_root_present": True,
241+ "checks": {"parquet": {"ok": True, "path": "/…/parquet"}, "sqlite": {"ok": True, "path": "/…/hfmd.db"},
242+ "redis": {"ok": True}, "duckdb": {"ok": True, "file": "AAPL_1day.parquet", "rows": 4200, "ms": 3.1}}}}}},
243+ 503: {"description": "Degraded (same body, `status: degraded`)"}})
244+def health(request: Request):
245+ request.state.quota_exempt = True
246+ body, status = _health.report(__version__)
247+ return JSONResponse(body, status_code=status, headers={"Cache-Control": "no-store"})
211 248
212 249
213 @app.get("/v1/status", tags=["meta"]) 250 @app.get("/v1/status", tags=["meta"])
214−def status(): 251+def status(request: Request):
215 """Dataset inventory: every asset type, timeframe and adjustment with 252 """Dataset inventory: every asset type, timeframe and adjustment with
216− the number of instruments currently available.""" 253+ the number of instruments currently available. Not charged against the rows quota."""
254+ request.state.quota_exempt = True
255+
217 def build(): 256 def build():
218 out = {} 257 out = {}
219 for asset in BAR_TYPES: 258 for asset in BAR_TYPES:
@@ -265,102 +304,96 @@ def tickers(asset: str,
265 # ------------------------------------------------------------------------------ 304 # ------------------------------------------------------------------------------
266 305
267 306
268−@app.get("/v1/bars/{asset}/{ticker}", tags=["bars"]) 307+@app.get("/v1/bars/{asset}/{ticker}", tags=["bars"],
269−def bars(asset: str, ticker: str, 308+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
309+def bars(request: Request, asset: str, ticker: str,
270 timeframe: str = Query("1day", description="1min|5min|30min|1hour|1day"), 310 timeframe: str = Query("1day", description="1min|5min|30min|1hour|1day"),
271 adjustment: str | None = Query(None, description="Default depends on asset type"), 311 adjustment: str | None = Query(None, description="Default depends on asset type"),
272− start: str | None = Query(None, description="ISO date/datetime lower bound"), 312+ start: str | None = Query(None, description="Lower bound (inclusive). " + _RANGE_DOC),
273− end: str | None = Query(None, description="ISO date/datetime upper bound"), 313+ end: str | None = Query(None, description="Upper bound (inclusive), must be ≥ `start`. " + _RANGE_DOC),
274 order: str = Query("asc", pattern="^(asc|desc)$"), 314 order: str = Query("asc", pattern="^(asc|desc)$"),
275− limit: int = Query(5_000, ge=1), 315+ limit: int = Query(5_000, ge=1, description=_LIMIT_DOC),
276 format: str = Query("json", pattern="^(json|csv)$")): 316 format: str = Query("json", pattern="^(json|csv)$")):
277 """OHLCV bars for one instrument. Daily futures bars also carry open 317 """OHLCV bars for one instrument. Daily futures bars also carry open
278− interest. `format=csv` allows bulk extraction (up to 2M rows).""" 318+ interest. `format=csv` allows bulk extraction (up to 2M rows).
319+
320+ Responses whose `end` is strictly before today are immutable history and carry
321+ `Cache-Control: public, max-age=86400`."""
279 adj = _validate(asset, timeframe, adjustment) 322 adj = _validate(asset, timeframe, adjustment)
280 idx = ticker_index(asset, timeframe, adj) 323 idx = ticker_index(asset, timeframe, adj)
281 path = idx.get(ticker.upper()) 324 path = idx.get(ticker.upper())
282 if not path: 325 if not path:
283− raise HTTPException(404, f"Ticker '{ticker.upper()}' not found in " 326+ raise ApiError(404, "TICKER_NOT_FOUND", f"Ticker '{ticker.upper()}' not found in "
284− f"{asset}/{timeframe}/{adj}") 327+ f"{asset}/{timeframe}/{adj}")
285− con = db() 328+ lo, hi = parse_range(start, end)
286− conds, params = [], [] 329+ where_sql, params = _range_conditions(lo, hi)
287− if start: 330+ where = f"WHERE {where_sql}" if where_sql else ""
288− conds.append("datetime >= ?"); params.append(start) 331+ lim = bound_limit(limit, _hard_max(format), request)
289− if end: 332+ df = con().execute(
290− conds.append("datetime <= ?"); params.append(end) 333+ f"SELECT * FROM read_parquet(?) {where} ORDER BY datetime {order.upper()} LIMIT ?",
291− where = ("WHERE " + " AND ".join(conds)) if conds else "" 334+ [path, *params, lim]).df()
292− rel = con.sql( 335+ return _frame_response(df, format, _history_headers(hi))
293− f"SELECT * FROM read_parquet(?) {where} ORDER BY datetime {order.upper()}", 336+
294− params=[path, *params]) 337+
295− return _rows_to_response(rel, limit, format, "datetime") 338+@app.get("/v1/bars/{asset}", tags=["bars"],
296− 339+ responses={200: {"description": "Bars of the found tickers", "headers": _MISSING_HEADER}},
297− 340+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "TOO_MANY_TICKERS", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
298−@app.get("/v1/bars/{asset}", tags=["bars"]) 341+def bars_multi(request: Request, asset: str,
299−def bars_multi(asset: str, 342+ tickers: str = Query(..., description=f"Comma-separated list, e.g. AAPL,MSFT,TSLA (max {MAX_TICKERS}; "
300− tickers: str = Query(..., description="Comma-separated list, e.g. AAPL,MSFT,TSLA (max 50)"), 343+ "more → 400 TOO_MANY_TICKERS). " + _MISSING_DOC),
301 timeframe: str = Query("1day"), 344 timeframe: str = Query("1day"),
302 adjustment: str | None = Query(None), 345 adjustment: str | None = Query(None),
303− start: str | None = Query(None, description="ISO date/datetime lower bound"), 346+ start: str | None = Query(None, description="Lower bound (inclusive). " + _RANGE_DOC),
304− end: str | None = Query(None, description="ISO date/datetime upper bound"), 347+ end: str | None = Query(None, description="Upper bound (inclusive), must be ≥ `start`. " + _RANGE_DOC),
305 order: str = Query("asc", pattern="^(asc|desc)$"), 348 order: str = Query("asc", pattern="^(asc|desc)$"),
306− limit: int = Query(5_000, ge=1, description="Max rows PER TICKER"), 349+ limit: int = Query(5_000, ge=1, description="Max rows PER TICKER. The total (limit × tickers) is also "
350+ "capped by the endpoint ceiling and your tier's `max_rows_per_request`."),
307 format: str = Query("json", pattern="^(json|csv)$")): 351 format: str = Query("json", pattern="^(json|csv)$")):
308 """Bars for SEVERAL instruments in one call. Combine with `start`/`end` 352 """Bars for SEVERAL instruments in one call. Combine with `start`/`end`
309 to slice any precise window — e.g. 4 hours of 1-minute bars across a 353 to slice any precise window — e.g. 4 hours of 1-minute bars across a
310− whole watchlist. The row limit applies per ticker.""" 354+ whole watchlist. The row limit applies per ticker.
355+
356+ Unknown tickers do not fail the whole call: they are listed in the `X-Missing-Tickers`
357+ response header (404 only when none of them exists). Windows ending before today carry
358+ `Cache-Control: public, max-age=86400`."""
311 adj = _validate(asset, timeframe, adjustment) 359 adj = _validate(asset, timeframe, adjustment)
312 idx = ticker_index(asset, timeframe, adj) 360 idx = ticker_index(asset, timeframe, adj)
313− names = [t.strip().upper() for t in tickers.split(",") if t.strip()][:50] 361+ names = split_tickers(tickers)
314− if not names: 362+ found, _missing, extra = _resolve_tickers(idx, names, f"{asset}/{timeframe}/{adj}")
315− raise HTTPException(400, "No tickers given") 363+ lo, hi = parse_range(start, end)
316− missing = [t for t in names if t not in idx] 364+ where_sql, params = _range_conditions(lo, hi)
317− if missing: 365+ hard = _hard_max(format)
318− raise HTTPException(404, f"Not found in {asset}/{timeframe}/{adj}: " 366+ per_ticker = bound_limit(limit, hard, request)
319− f"{', '.join(missing)}") 367+ total = bound_limit(per_ticker * len(found), hard, request)
320− paths = [idx[t] for t in names] 368+ paths = [idx[t] for t in found]
321− con = db() 369+ df = run_topn(con(), paths, where_sql, params, "ticker", f"datetime {order.upper()}", per_ticker,
322− conds, params = [], [] 370+ order_by=f"ticker, datetime {order.upper()}", limit=total)
323− if start: 371+ return _frame_response(df, format, {**extra, **_history_headers(hi)})
324− conds.append("datetime >= ?"); params.append(start) 372+
325− if end: 373+
326− conds.append("datetime <= ?"); params.append(end) 374+@app.get("/v1/snapshot/{asset}", tags=["bars"],
327− where = ("WHERE " + " AND ".join(conds)) if conds else "" 375+ responses={200: {"description": "Last bar at or before `at` for each found ticker", "headers": _MISSING_HEADER}},
328− per_ticker = min(limit, MAX_LIMIT_CSV if format == "csv" else MAX_LIMIT_JSON) 376+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "TOO_MANY_TICKERS", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
329− rel = con.sql( 377+def snapshot(request: Request, asset: str,
330− f"SELECT * FROM read_parquet(?) {where} " 378+ tickers: str = Query(..., description=f"Comma-separated list, e.g. AAPL,MSFT,TSLA (max {MAX_TICKERS}). " + _MISSING_DOC),
331− f"QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY datetime " 379+ at: str = Query(..., description="Precise moment, e.g. 2024-06-03 10:35:00. " + _RANGE_DOC),
332− f"{order.upper()}) <= ? ORDER BY ticker, datetime {order.upper()}",
333− params=[paths, *params, per_ticker])
334− return _rows_to_response(rel, per_ticker * len(names), format, "datetime")
335−
336−
337−@app.get("/v1/snapshot/{asset}", tags=["bars"])
338−def snapshot(asset: str,
339− tickers: str = Query(..., description="Comma-separated list, e.g. AAPL,MSFT,TSLA (max 50)"),
340− at: str = Query(..., description="Precise moment, e.g. 2024-06-03 10:35:00"),
341 timeframe: str = Query("1min"), 380 timeframe: str = Query("1min"),
342 adjustment: str | None = Query(None), 381 adjustment: str | None = Query(None),
343 format: str = Query("json", pattern="^(json|csv)$")): 382 format: str = Query("json", pattern="^(json|csv)$")):
344 """Cross-sectional snapshot: for each requested instrument, the last bar 383 """Cross-sectional snapshot: for each requested instrument, the last bar
345 at or before the given moment — the state of a whole watchlist at one 384 at or before the given moment — the state of a whole watchlist at one
346− precise point in time.""" 385+ precise point in time. Unknown tickers are reported in `X-Missing-Tickers`."""
347 adj = _validate(asset, timeframe, adjustment) 386 adj = _validate(asset, timeframe, adjustment)
348 idx = ticker_index(asset, timeframe, adj) 387 idx = ticker_index(asset, timeframe, adj)
349− names = [t.strip().upper() for t in tickers.split(",") if t.strip()][:50] 388+ names = split_tickers(tickers)
350− if not names: 389+ found, _missing, extra = _resolve_tickers(idx, names, f"{asset}/{timeframe}/{adj}")
351− raise HTTPException(400, "No tickers given") 390+ at_dt = parse_datetime(at, "at")
352− missing = [t for t in names if t not in idx] 391+ if at_dt is None:
353− if missing: 392+ raise ApiError(400, "INVALID_PARAMETER", "at is required", details={"at": at})
354− raise HTTPException(404, f"Not found in {asset}/{timeframe}/{adj}: " 393+ paths = [idx[t] for t in found]
355− f"{', '.join(missing)}") 394+ df = run_topn(con(), paths, "datetime <= ?", [at_dt], "ticker", "datetime DESC", 1,
356− paths = [idx[t] for t in names] 395+ order_by="ticker", limit=len(found))
357− con = db() 396+ return _frame_response(df, format, {**extra, **_history_headers(at_dt)})
358− rel = con.sql(
359− "SELECT * FROM read_parquet(?) WHERE datetime <= ? "
360− "QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY datetime DESC) = 1 "
361− "ORDER BY ticker",
362− params=[paths, at])
363− return _rows_to_response(rel, len(names), format, "datetime")
364 397
365 398
366 # ------------------------------------------------------------------------------ 399 # ------------------------------------------------------------------------------
@@ -381,10 +414,10 @@ def opt_tickers(quarter: str | None = Query(None, description="e.g. 2024_q4; def
381 """Underlyings available in a given options quarter.""" 414 """Underlyings available in a given options quarter."""
382 quarters = options_quarters() 415 quarters = options_quarters()
383 if not quarters: 416 if not quarters:
384− raise HTTPException(503, "Options dataset not yet available") 417+ raise ApiError(503, "SERVICE_UNAVAILABLE", "Options dataset not yet available")
385 q = quarter or quarters[-1] 418 q = quarter or quarters[-1]
386 if q not in quarters: 419 if q not in quarters:
387− raise HTTPException(404, f"Unknown quarter '{q}'") 420+ raise ApiError(404, "NOT_FOUND", f"Unknown quarter '{q}'")
388 421
389 def build(): 422 def build():
390 return sorted({p.stem.split("_")[0].upper() 423 return sorted({p.stem.split("_")[0].upper()
@@ -396,32 +429,39 @@ def opt_tickers(quarter: str | None = Query(None, description="e.g. 2024_q4; def
396 return {"quarter": q, "count": len(names[:limit]), "tickers": names[:limit]} 429 return {"quarter": q, "count": len(names[:limit]), "tickers": names[:limit]}
397 430
398 431
399−@app.get("/v1/options/chain/{ticker}", tags=["options"]) 432+def _no_options(ticker: str) -> ApiError:
400−def opt_chain(ticker: str, 433+ return ApiError(404, "OPTIONS_UNAVAILABLE", f"No options data for '{ticker.upper()}'")
434+
435+
436+@app.get("/v1/options/chain/{ticker}", tags=["options"],
437+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
438+def opt_chain(request: Request, ticker: str,
401 trade_date: str | None = Query(None, description="yyyy-mm-dd; default latest available"), 439 trade_date: str | None = Query(None, description="yyyy-mm-dd; default latest available"),
402− expiry: str | None = Query(None, description="Filter on expiry date"), 440+ expiry: str | None = Query(None, description="Filter on expiry date (yyyy-mm-dd)"),
403 call_put: str | None = Query(None, pattern="^(c|p)$"), 441 call_put: str | None = Query(None, pattern="^(c|p)$"),
404 strike_min: float | None = None, 442 strike_min: float | None = None,
405 strike_max: float | None = None, 443 strike_max: float | None = None,
406 min_volume: float | None = None, 444 min_volume: float | None = None,
407− limit: int = Query(20_000, ge=1), 445+ limit: int = Query(20_000, ge=1, description=_LIMIT_DOC),
408 format: str = Query("json", pattern="^(json|csv)$")): 446 format: str = Query("json", pattern="^(json|csv)$")):
409 """Full end-of-day option chain for an underlying: quotes, bid/ask implied 447 """Full end-of-day option chain for an underlying: quotes, bid/ask implied
410 volatility, open interest, volume and Greeks (delta, gamma, vega, theta, rho).""" 448 volatility, open interest, volume and Greeks (delta, gamma, vega, theta, rho)."""
411 files = options_files(ticker) 449 files = options_files(ticker)
412 if not files: 450 if not files:
413− raise HTTPException(404, f"No options data for '{ticker.upper()}'") 451+ raise _no_options(ticker)
414− con = db() 452+ c = con()
415− if trade_date is None: 453+ td = parse_date(trade_date, "trade_date")
416− trade_date = con.execute( 454+ if td is None:
417− "SELECT max(trade_date) FROM read_parquet(?)", [files[-1]] 455+ latest = c.execute("SELECT max(trade_date) FROM read_parquet(?)", [files[-1]]).fetchone()[0]
418− ).fetchone()[0] 456+ td_s = str(latest)
419 files_q = [files[-1]] 457 files_q = [files[-1]]
420 else: 458 else:
459+ td_s = td.isoformat()
421 files_q = files 460 files_q = files
422− conds, params = ["trade_date = ?"], [str(trade_date)] 461+ conds, params = ["trade_date = ?"], [td_s]
423− if expiry: 462+ exp = parse_date(expiry, "expiry")
424− conds.append("expiry = ?"); params.append(expiry) 463+ if exp is not None:
464+ conds.append("expiry = ?"); params.append(exp.isoformat())
425 if call_put: 465 if call_put:
426 conds.append("call_put = ?"); params.append(call_put) 466 conds.append("call_put = ?"); params.append(call_put)
427 if strike_min is not None: 467 if strike_min is not None:
@@ -430,49 +470,54 @@ def opt_chain(ticker: str,
430 conds.append("strike <= ?"); params.append(strike_max) 470 conds.append("strike <= ?"); params.append(strike_max)
431 if min_volume is not None: 471 if min_volume is not None:
432 conds.append("volume >= ?"); params.append(min_volume) 472 conds.append("volume >= ?"); params.append(min_volume)
433− rel = con.sql( 473+ lim = bound_limit(limit, _hard_max(format), request)
474+ df = c.execute(
434 f"SELECT * FROM read_parquet(?) WHERE {' AND '.join(conds)} " 475 f"SELECT * FROM read_parquet(?) WHERE {' AND '.join(conds)} "
435− f"ORDER BY expiry, strike, call_put", 476+ f"ORDER BY expiry, strike, call_put LIMIT ?",
436− params=[files_q, *params]) 477+ [files_q, *params, lim]).df()
437− return _rows_to_response(rel, limit, format, "expiry") 478+ return _frame_response(df, format)
438 479
439 480
440−@app.get("/v1/options/expirations/{ticker}", tags=["options"]) 481+@app.get("/v1/options/expirations/{ticker}", tags=["options"],
482+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
441 def opt_expirations(ticker: str, 483 def opt_expirations(ticker: str,
442 trade_date: str | None = Query(None, description="yyyy-mm-dd")): 484 trade_date: str | None = Query(None, description="yyyy-mm-dd")):
443 """Available expiry dates (optionally as of one trade date).""" 485 """Available expiry dates (optionally as of one trade date)."""
444 files = options_files(ticker) 486 files = options_files(ticker)
445 if not files: 487 if not files:
446− raise HTTPException(404, f"No options data for '{ticker.upper()}'") 488+ raise _no_options(ticker)
447− con = db() 489+ c = con()
448− if trade_date: 490+ td = parse_date(trade_date, "trade_date")
449− rows = con.execute( 491+ if td is not None:
492+ rows = c.execute(
450 "SELECT DISTINCT expiry FROM read_parquet(?) WHERE trade_date = ? ORDER BY expiry", 493 "SELECT DISTINCT expiry FROM read_parquet(?) WHERE trade_date = ? ORDER BY expiry",
451− [files, trade_date]).fetchall() 494+ [files, td.isoformat()]).fetchall()
452 else: 495 else:
453− rows = con.execute( 496+ rows = c.execute(
454 "SELECT DISTINCT expiry FROM read_parquet(?) ORDER BY expiry", 497 "SELECT DISTINCT expiry FROM read_parquet(?) ORDER BY expiry",
455 [files[-1]]).fetchall() 498 [files[-1]]).fetchall()
456 return {"ticker": ticker.upper(), "expirations": [str(r[0]) for r in rows]} 499 return {"ticker": ticker.upper(), "expirations": [str(r[0]) for r in rows]}
457 500
458 501
459−@app.get("/v1/options/history/{ticker}", tags=["options"]) 502+@app.get("/v1/options/history/{ticker}", tags=["options"],
460−def opt_history(ticker: str, 503+ openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
504+def opt_history(request: Request, ticker: str,
461 strike: float = Query(...), 505 strike: float = Query(...),
462 expiry: str = Query(..., description="yyyy-mm-dd"), 506 expiry: str = Query(..., description="yyyy-mm-dd"),
463 call_put: str = Query(..., pattern="^(c|p)$"), 507 call_put: str = Query(..., pattern="^(c|p)$"),
464− limit: int = Query(5_000, ge=1), 508+ limit: int = Query(5_000, ge=1, description=_LIMIT_DOC),
465 format: str = Query("json", pattern="^(json|csv)$")): 509 format: str = Query("json", pattern="^(json|csv)$")):
466 """Daily time series for one specific contract across its whole life.""" 510 """Daily time series for one specific contract across its whole life."""
467 files = options_files(ticker) 511 files = options_files(ticker)
468 if not files: 512 if not files:
469− raise HTTPException(404, f"No options data for '{ticker.upper()}'") 513+ raise _no_options(ticker)
470− con = db() 514+ exp = parse_date(expiry, "expiry")
471− rel = con.sql( 515+ lim = bound_limit(limit, _hard_max(format), request)
516+ df = con().execute(
472 "SELECT * FROM read_parquet(?) WHERE strike = ? AND expiry = ? AND call_put = ? " 517 "SELECT * FROM read_parquet(?) WHERE strike = ? AND expiry = ? AND call_put = ? "
473− "ORDER BY trade_date", 518+ "ORDER BY trade_date LIMIT ?",
474− params=[files, strike, expiry, call_put]) 519+ [files, strike, exp.isoformat(), call_put, lim]).df()
475− return _rows_to_response(rel, limit, format, "trade_date") 520+ return _frame_response(df, format)
476 521
477 522
478 # ------------------------------------------------------------------------------ 523 # ------------------------------------------------------------------------------
@@ -498,44 +543,51 @@ for _mod in V2_MODULES:
498 except Exception as e: # pragma: no cover 543 except Exception as e: # pragma: no cover
499 _log.exception("module %s failed to load: %s", _mod, e) 544 _log.exception("module %s failed to load: %s", _mod, e)
500 545
546+# Request id + JSON access log + security headers: outermost layer, mounted after every other middleware.
547+_http.install(app)
548+
549+
550+# ------------------------------------------------------------------------------
551+# OpenAPI document (ETag + Cache-Control) and the interactive consoles
552+# ------------------------------------------------------------------------------
553+
554+_OPENAPI_CACHE = "public, max-age=300"
555+
556+
557+def _openapi_body() -> tuple[bytes, str]:
558+ def build():
559+ import json
560+ body = json.dumps(app.openapi(), separators=(",", ":"), ensure_ascii=False).encode()
561+ return body, '"' + hashlib.sha256(body).hexdigest()[:32] + '"'
562+ return cached("openapi|json", build, ttl=3600)
501 563
502−# ETag on the OpenAPI document so the docs' If-None-Match revalidation is cheap
503−import hashlib as _hashlib
504 564
505−@app.middleware("http") 565+@app.get("/openapi.json", include_in_schema=False)
506−async def _openapi_etag(request, call_next): 566+def openapi_json(request: Request):
507− response = await call_next(request) 567+ body, etag = _openapi_body()
508− if request.url.path == "/openapi.json" and response.status_code == 200: 568+ if etag in [t.strip() for t in (request.headers.get("if-none-match") or "").split(",")]:
509− body = b"".join([chunk async for chunk in response.body_iterator]) 569+ return Response(status_code=304, headers={"ETag": etag, "Cache-Control": _OPENAPI_CACHE})
510− etag = '"' + _hashlib.sha256(body).hexdigest()[:32] + '"' 570+ return Response(body, media_type="application/json", headers={"ETag": etag, "Cache-Control": _OPENAPI_CACHE})
511− if request.headers.get("if-none-match") == etag: 571+
512− return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "public, max-age=300"}) 572+
513− return Response(body, status_code=200, media_type="application/json", 573+@app.get("/swagger", include_in_schema=False)
514− headers={"ETag": etag, "Cache-Control": "public, max-age=300"}) 574+def swagger_ui():
515− return response 575+ return get_swagger_ui_html(openapi_url="/openapi.json", title=f"{app.title} — Swagger UI",
576+ swagger_favicon_url="/favicon.svg")
577+
578+
579+@app.get("/redoc", include_in_schema=False)
580+def redoc_ui():
581+ return get_redoc_html(openapi_url="/openapi.json", title=f"{app.title} — ReDoc", redoc_favicon_url="/favicon.svg")
516 582
517 583
518 # ------------------------------------------------------------------------------ 584 # ------------------------------------------------------------------------------
519−# Static React platform (mounted last so it doesn't shadow the API) + SPA fallback 585+# Static React platform (mounted last so it doesn't shadow the API) + SPA fallback — see core/spa.py
520 # ------------------------------------------------------------------------------ 586 # ------------------------------------------------------------------------------
521 587
522−if WEB_DIST.is_dir(): 588+_spa.install(app, WEB_DIST)
523− app.mount("/assets", StaticFiles(directory=str(WEB_DIST / "assets")), name="assets")
524−
525− @app.get("/{path:path}", include_in_schema=False)
526− def spa(path: str):
527− if path.startswith(("v1/", "health", "openapi", "docs/", "redoc")) and not path.startswith("docs/"):
528− raise HTTPException(404, "Not found")
529− root = str(WEB_DIST.resolve())
530− candidate = (WEB_DIST / path).resolve()
531− if path and str(candidate).startswith(root):
532− # exact file, prerendered shell (<route>.html or <route>/index.html), else the SPA entry
533− for c in (candidate, candidate.with_suffix(".html") if candidate.suffix == "" else None, candidate / "index.html"):
534− if c is not None and c.is_file():
535− return FileResponse(c)
536− return FileResponse(WEB_DIST / "index.html")
537 589
538 590
539 if __name__ == "__main__": 591 if __name__ == "__main__":
540 import uvicorn 592 import uvicorn
541− uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("HFMD_PORT", 8090))) 593+ uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("HFMD_PORT", 8090)), access_log=False)
modified hfmarketdata/api/openapi.py +2 −1
@@ -33,6 +33,7 @@ RATE_HEADERS = {
33 "X-RateLimit-Remaining-Rows": {"schema": {"type": "integer"}, "description": "Data rows left in the current window."}, 33 "X-RateLimit-Remaining-Rows": {"schema": {"type": "integer"}, "description": "Data rows left in the current window."},
34 "X-RateLimit-Reset": {"schema": {"type": "integer"}, "description": "Unix timestamp (seconds) when the window resets."}, 34 "X-RateLimit-Reset": {"schema": {"type": "integer"}, "description": "Unix timestamp (seconds) when the window resets."},
35 "X-Row-Count": {"schema": {"type": "integer"}, "description": "Rows returned by this response."}, 35 "X-Row-Count": {"schema": {"type": "integer"}, "description": "Rows returned by this response."},
36+ "X-Request-ID": {"schema": {"type": "string"}, "description": "Correlation id of the request (echoed when you send one); quote it when reporting a problem."},
36 } 37 }
37 38
38 ERROR_SCHEMA = { 39 ERROR_SCHEMA = {
@@ -52,7 +53,7 @@ ERROR_SCHEMA = {
52 "docs": docs_link("CONTRACT_NOT_FOUND")}, "detail": "No data for contract ESZ19"}, 53 "docs": docs_link("CONTRACT_NOT_FOUND")}, "detail": "No data for contract ESZ19"},
53 } 54 }
54 55
55−STATUS_FOR = {"INVALID_PARAMETER": 400, "VALIDATION_ERROR": 422, "NOT_FOUND": 404, "TICKER_NOT_FOUND": 404, 56+STATUS_FOR = {"INVALID_PARAMETER": 400, "TOO_MANY_TICKERS": 400, "VALIDATION_ERROR": 422, "NOT_FOUND": 404, "TICKER_NOT_FOUND": 404,
56 "ASSET_NOT_FOUND": 404, "INVALID_CONTRACT_SYMBOL": 400, "CONTRACT_NOT_FOUND": 404, "ROOT_NOT_FOUND": 404, 57 "ASSET_NOT_FOUND": 404, "INVALID_CONTRACT_SYMBOL": 400, "CONTRACT_NOT_FOUND": 404, "ROOT_NOT_FOUND": 404,
57 "OPTIONS_UNAVAILABLE": 503, "INVALID_API_KEY": 401, "AUTH_REQUIRED": 401, "FORBIDDEN": 403, 58 "OPTIONS_UNAVAILABLE": 503, "INVALID_API_KEY": 401, "AUTH_REQUIRED": 401, "FORBIDDEN": 403,
58 "RATE_LIMIT_EXCEEDED": 429, "ROW_LIMIT_EXCEEDED": 400, "CONFLICT": 409, "INTERNAL_ERROR": 500, 59 "RATE_LIMIT_EXCEEDED": 429, "ROW_LIMIT_EXCEEDED": 400, "CONFLICT": 409, "INTERNAL_ERROR": 500,
added tests/test_api_hardening.py +319 −0
@@ -0,0 +1,319 @@
1+"""Chantier api — strict v1 validation (400 instead of 500), partial multi-ticker results, tier-bounded limits,
2+HTTP hygiene (gzip, X-Request-ID, security headers, caching), /health, Swagger moved to /swagger."""
3+from __future__ import annotations
4+
5+import json
6+import logging
7+
8+import pytest
9+
10+# ----------------------------------------------------------------------------------------------------- validation
11+
12+
13+@pytest.mark.parametrize("url", [
14+ "/v1/bars/stock/AAPL?timeframe=1day&start=bad",
15+ "/v1/bars/stock/AAPL?timeframe=1day&end=2024-13-45",
16+ "/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01T25:00:00",
17+ "/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&start=nope",
18+ "/v1/snapshot/stock?tickers=AAPL&at=yesterday",
19+ "/v1/options/chain/AAPL?trade_date=2025-04-0x",
20+ "/v1/options/chain/AAPL?expiry=soon",
21+ "/v1/options/expirations/AAPL?trade_date=x",
22+ "/v1/options/history/AAPL?strike=200&expiry=x&call_put=c",
23+])
24+def test_invalid_dates_are_400(client, url):
25+ r = client.get(url)
26+ assert r.status_code == 400, r.text
27+ body = r.json()
28+ assert body["error"]["code"] == "INVALID_PARAMETER"
29+ assert body["detail"] # legacy field
30+ assert "Traceback" not in r.text
31+
32+
33+def test_end_before_start_is_400(client):
34+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-06-10&end=2024-06-01")
35+ assert r.status_code == 400
36+ assert r.json()["error"]["code"] == "INVALID_PARAMETER"
37+ assert "start" in r.json()["error"]["details"]
38+
39+
40+@pytest.mark.parametrize("start,end", [
41+ ("2024-06-03", "2024-06-04"),
42+ ("2024-06-03 09:30:00", "2024-06-03 09:34:59"),
43+ ("2024-06-03T09:30:00", "2024-06-03T09:34:59Z"),
44+ ("2024-06-03T09:30:00+00:00", None),
45+ ("2024-06-03T05:30:00-04:00", "2024-06-04"),
46+])
47+def test_accepted_date_forms(client, start, end):
48+ from urllib.parse import quote
49+ q = f"start={quote(start)}" + (f"&end={quote(end)}" if end else "")
50+ r = client.get(f"/v1/bars/stock/AAPL?timeframe=1day&{q}")
51+ assert r.status_code == 200, r.text
52+
53+
54+def test_window_is_honoured_and_typed(client):
55+ r = client.get("/v1/bars/stock/AAPL?timeframe=1min&start=2025-06-30 09:30:00&end=2025-06-30 09:34:59&adjustment=UNADJUSTED")
56+ assert r.status_code == 200
57+ rows = r.json()["data"]
58+ assert len(rows) == 5
59+ assert rows[0]["datetime"].startswith("2025-06-30 09:30") and rows[-1]["datetime"].startswith("2025-06-30 09:34")
60+
61+
62+def test_too_many_tickers_is_400(client):
63+ names = ",".join(f"T{i}" for i in range(51))
64+ r = client.get(f"/v1/bars/stock?tickers={names}&timeframe=1day")
65+ assert r.status_code == 400
66+ assert r.json()["error"]["code"] == "TOO_MANY_TICKERS"
67+ r = client.get(f"/v1/snapshot/stock?tickers={names}&at=2025-06-30 10:00:00")
68+ assert r.status_code == 400 and r.json()["error"]["code"] == "TOO_MANY_TICKERS"
69+
70+
71+def test_fifty_tickers_ok_dedup(client):
72+ names = ",".join(["AAPL", "aapl", "MSFT"] + [f"T{i}" for i in range(47)]) # 49 distinct
73+ r = client.get(f"/v1/bars/stock?tickers={names}&timeframe=1day&limit=1")
74+ assert r.status_code == 200
75+ assert sorted({row["ticker"] for row in r.json()["data"]}) == ["AAPL", "MSFT"]
76+
77+
78+def test_empty_tickers_is_400(client):
79+ r = client.get("/v1/bars/stock?tickers=,,&timeframe=1day")
80+ assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
81+
82+
83+# -------------------------------------------------------------------------------------------- partial results
84+
85+
86+def test_partially_missing_tickers_200_with_header(client):
87+ r = client.get("/v1/bars/stock?tickers=AAPL,NOPE,MSFT,ZZZ&timeframe=1day&limit=2")
88+ assert r.status_code == 200, r.text
89+ assert r.headers["X-Missing-Tickers"] == "NOPE,ZZZ"
90+ body = r.json()
91+ assert body["count"] == 4 and {row["ticker"] for row in body["data"]} == {"AAPL", "MSFT"}
92+ assert r.headers["X-Row-Count"] == "4"
93+
94+
95+def test_all_missing_tickers_404(client):
96+ r = client.get("/v1/bars/stock?tickers=NOPE,ZZZ&timeframe=1day")
97+ assert r.status_code == 404
98+ assert r.json()["error"]["code"] == "TICKER_NOT_FOUND"
99+ assert r.json()["error"]["details"]["missing"] == ["NOPE", "ZZZ"]
100+
101+
102+def test_no_missing_header_when_all_found(client):
103+ r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&limit=1")
104+ assert r.status_code == 200 and "X-Missing-Tickers" not in r.headers
105+
106+
107+def test_snapshot_partial(client):
108+ r = client.get("/v1/snapshot/stock?tickers=AAPL,NOPE&at=2025-06-30 10:00:00&adjustment=UNADJUSTED")
109+ assert r.status_code == 200
110+ assert r.headers["X-Missing-Tickers"] == "NOPE"
111+ body = r.json()
112+ assert body["count"] == 1 and body["data"][0]["ticker"] == "AAPL"
113+ assert body["data"][0]["datetime"].startswith("2025-06-30 10:00")
114+
115+
116+def test_multi_per_ticker_limit_and_order(client):
117+ r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&limit=3&order=desc")
118+ assert r.status_code == 200
119+ rows = r.json()["data"]
120+ assert len(rows) == 6
121+ aapl = [x["datetime"] for x in rows if x["ticker"] == "AAPL"]
122+ assert aapl == sorted(aapl, reverse=True) and aapl[0].startswith("2025-06-30")
123+
124+
125+# ------------------------------------------------------------------------------------------------- limits
126+
127+
128+def test_limit_bounded_by_tier_max_rows(client):
129+ """Keyless tier: 5 000 rows per request. 6 tickers × 1 950 one-minute bars would be 11 700 rows."""
130+ r = client.get("/v1/bars/stock?tickers=AAPL,MSFT,SMCP,SHAK,GOOG,GOOGL&timeframe=1min&limit=5000&adjustment=UNADJUSTED")
131+ assert r.status_code == 200
132+ assert r.headers["X-Row-Count"] == "5000" and r.json()["count"] == 5000
133+
134+
135+def test_single_limit_bounded_by_tier_max_rows(client_hu, client):
136+ from core.params import bound_limit
137+
138+ class _Req:
139+ class state:
140+ max_rows = 100
141+ assert bound_limit(5000, 50_000, _Req()) == 100
142+ assert bound_limit(50, 50_000, _Req()) == 50
143+
144+ class _NoTier:
145+ class state:
146+ pass
147+ assert bound_limit(5_000_000, 50_000, _NoTier()) == 50_000
148+ assert bound_limit(10, 50_000, None) == 10
149+
150+
151+# --------------------------------------------------------------------------------------------- HTTP hygiene
152+
153+
154+def test_gzip_when_accepted(client):
155+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=500", headers={"Accept-Encoding": "gzip"})
156+ assert r.status_code == 200
157+ assert r.headers.get("Content-Encoding") == "gzip"
158+ assert r.json()["count"] == 500 # httpx decodes transparently
159+
160+
161+def test_no_gzip_for_small_bodies(client):
162+ r = client.get("/health", headers={"Accept-Encoding": "gzip"})
163+ assert r.status_code in (200, 503)
164+ # /health is above 1 KiB only with many checks; the point is that gzip is negotiated, not forced
165+ r = client.get("/v1/options/quarters", headers={"Accept-Encoding": "gzip"})
166+ assert "Content-Encoding" not in r.headers
167+
168+
169+def test_request_id_generated_and_echoed(client):
170+ r = client.get("/v1/options/quarters")
171+ rid = r.headers["X-Request-ID"]
172+ assert 8 <= len(rid) <= 64
173+ r2 = client.get("/v1/options/quarters", headers={"X-Request-ID": "trace-abc.123"})
174+ assert r2.headers["X-Request-ID"] == "trace-abc.123"
175+ r3 = client.get("/v1/options/quarters", headers={"X-Request-ID": "bad id with spaces"})
176+ assert r3.headers["X-Request-ID"] != "bad id with spaces"
177+
178+
179+def test_request_id_on_errors_too(client):
180+ r = client.get("/v1/bars/stock/NOPE?timeframe=1day")
181+ assert r.status_code == 404 and r.headers.get("X-Request-ID")
182+
183+
184+def test_access_log_line_is_json(client, caplog):
185+ with caplog.at_level(logging.INFO, logger="hfmarketdata.access"):
186+ client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=7", headers={"X-Request-ID": "log-test-1"})
187+ lines = [rec.getMessage() for rec in caplog.records if rec.name == "hfmarketdata.access"]
188+ assert lines, "no access log line"
189+ rec = json.loads(lines[-1])
190+ assert rec["request_id"] == "log-test-1" and rec["method"] == "GET" and rec["path"] == "/v1/bars/stock/AAPL"
191+ assert rec["status"] == 200 and rec["rows"] == 7 and rec["bytes"] > 0 and rec["duration_ms"] >= 0
192+ assert "principal" in rec and rec["query"].startswith("timeframe=1day")
193+
194+
195+def test_security_headers_on_json(client):
196+ r = client.get("/v1/options/quarters")
197+ assert r.headers["Strict-Transport-Security"] == "max-age=31536000; includeSubDomains"
198+ assert r.headers["X-Content-Type-Options"] == "nosniff"
199+ assert r.headers["Referrer-Policy"] == "strict-origin-when-cross-origin"
200+ assert r.headers["X-Frame-Options"] == "DENY"
201+ assert "Content-Security-Policy" not in r.headers
202+
203+
204+def test_swagger_and_redoc_moved(client):
205+ r = client.get("/swagger")
206+ assert r.status_code == 200 and "swagger-ui" in r.text
207+ assert "Content-Security-Policy" in r.headers and "cdn.jsdelivr.net" in r.headers["Content-Security-Policy"]
208+ assert "X-Frame-Options" not in r.headers
209+ r = client.get("/redoc")
210+ assert r.status_code == 200 and "redoc" in r.text.lower()
211+
212+
213+def test_docs_is_not_swagger(client):
214+ """Without a built front-end the shell is absent (404 JSON) — but never Swagger UI."""
215+ r = client.get("/docs")
216+ assert "swagger-ui" not in r.text
217+ assert r.status_code == 404
218+
219+
220+def test_openapi_etag_and_cache(client):
221+ r = client.get("/openapi.json")
222+ assert r.status_code == 200
223+ etag = r.headers["ETag"]
224+ assert etag.startswith('"') and r.headers["Cache-Control"].startswith("public")
225+ spec = r.json()
226+ assert spec["openapi"] == "3.1.0" and "/v1/bars/{asset}" in spec["paths"]
227+ r2 = client.get("/openapi.json", headers={"If-None-Match": etag})
228+ assert r2.status_code == 304 and r2.headers["ETag"] == etag
229+
230+
231+def test_openapi_documents_new_errors_and_headers(client):
232+ spec = client.get("/openapi.json").json()
233+ op = spec["paths"]["/v1/bars/{asset}"]["get"]
234+ assert "X-Missing-Tickers" in op["responses"]["200"]["headers"]
235+ assert "X-Request-ID" in op["responses"]["200"]["headers"]
236+ assert "TOO_MANY_TICKERS" in op["responses"]["400"]["description"]
237+ assert "TOO_MANY_TICKERS" in spec["components"]["schemas"]["Error"]["properties"]["error"]["properties"]["code"]["enum"]
238+
239+
240+def test_history_cache_control(client):
241+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01&end=2024-01-31")
242+ assert r.status_code == 200 and r.headers["Cache-Control"] == "public, max-age=86400"
243+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01") # open-ended → not cacheable
244+ assert "Cache-Control" not in r.headers
245+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01&end=2999-01-01")
246+ assert "Cache-Control" not in r.headers
247+ r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&start=2024-01-01&end=2024-01-31")
248+ assert r.headers["Cache-Control"] == "public, max-age=86400"
249+
250+
251+def test_login_redirects_to_signin(client):
252+ r = client.get("/login", follow_redirects=False)
253+ assert r.status_code == 301 and r.headers["Location"] == "/signin"
254+
255+
256+# ------------------------------------------------------------------------------------------------- /health
257+
258+
259+def test_health_extended(client):
260+ r = client.get("/health")
261+ assert r.status_code == 200, r.text
262+ body = r.json()
263+ assert body["status"] == "ok" and body["data_root_present"] is True
264+ checks = body["checks"]
265+ assert checks["parquet"]["ok"] and checks["sqlite"]["ok"] and checks["redis"]["ok"] and checks["duckdb"]["ok"]
266+ assert checks["redis"].get("backend") == "fakeredis"
267+ assert checks["duckdb"]["rows"] > 0 and checks["duckdb"]["file"].endswith(".parquet")
268+ assert "X-RateLimit-Limit-Requests" not in r.headers # exempt from quotas
269+ assert r.headers["Cache-Control"] == "no-store"
270+
271+
272+def test_health_degraded_is_503(client, monkeypatch):
273+ from core import health
274+ monkeypatch.setattr(health, "_check_sqlite", lambda: {"ok": False, "error": "boom"})
275+ r = client.get("/health")
276+ assert r.status_code == 503 and r.json()["status"] == "degraded"
277+ assert r.json()["checks"]["sqlite"]["error"] == "boom"
278+
279+
280+def test_status_is_quota_exempt(client):
281+ r = client.get("/v1/status")
282+ assert r.status_code == 200
283+ # the request is counted, the rows are not
284+ assert r.headers.get("X-RateLimit-Remaining-Rows") == r.headers.get("X-RateLimit-Limit-Rows")
285+
286+
287+# --------------------------------------------------------------------------------------- v1 errors unchanged
288+
289+
290+def test_unknown_asset_404_asset_code(client):
291+ r = client.get("/v1/bars/bond/XYZ")
292+ assert r.status_code == 404 and r.json()["error"]["code"] == "ASSET_NOT_FOUND"
293+
294+
295+def test_unknown_timeframe_400(client):
296+ r = client.get("/v1/bars/stock/AAPL?timeframe=2min")
297+ assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
298+
299+
300+def test_duckdb_error_never_leaks_traceback(client, monkeypatch):
301+ import duckdb
302+ import main
303+
304+ class _Boom:
305+ def execute(self, *a, **k):
306+ raise duckdb.ConversionException("Conversion Error: invalid timestamp field format")
307+
308+ monkeypatch.setattr(main, "con", lambda: _Boom())
309+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day")
310+ assert r.status_code == 400
311+ assert r.json()["error"]["code"] == "INVALID_PARAMETER" and "Traceback" not in r.text
312+
313+ class _IO:
314+ def execute(self, *a, **k):
315+ raise duckdb.IOException("IO Error: No such file")
316+
317+ monkeypatch.setattr(main, "con", lambda: _IO())
318+ r = client.get("/v1/bars/stock/AAPL?timeframe=1day")
319+ assert r.status_code == 503 and r.json()["error"]["code"] == "SERVICE_UNAVAILABLE"
added tests/test_spa.py +109 −0
@@ -0,0 +1,109 @@
1+"""SPA fallback (core/spa.py) on a bare FastAPI app with a fake Vite dist: known routes → 200 shell, unknown → 404 shell,
2+real files served, hashed assets immutable + gzipped, API paths never answered by the shell."""
3+from __future__ import annotations
4+
5+import pytest
6+from fastapi import FastAPI
7+from fastapi.testclient import TestClient
8+from starlette.middleware.gzip import GZipMiddleware
9+
10+
11+@pytest.fixture(scope="module")
12+def spa_client(tmp_path_factory, app):
13+ from core import errors, http, spa
14+ dist = tmp_path_factory.mktemp("dist")
15+ (dist / "index.html").write_text("<!doctype html><html><body><div id=root></div><script>/*shell*/</script></body></html>" + " " * 1500)
16+ (dist / "assets").mkdir()
17+ (dist / "assets" / "index-abc123.js").write_text("console.log('hfmd');" * 200)
18+ (dist / "favicon.svg").write_text("<svg xmlns='http://www.w3.org/2000/svg'></svg>")
19+ (dist / "docs").mkdir()
20+ (dist / "docs" / "index.html").write_text("<!doctype html><html><body>prerendered docs</body></html>")
21+ (dist / "limits.html").write_text("<!doctype html><html><body>prerendered limits</body></html>")
22+ a = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
23+ errors.install(a)
24+ a.add_middleware(GZipMiddleware, minimum_size=1024)
25+
26+ @a.get("/v1/ping")
27+ def ping():
28+ return {"ok": True}
29+
30+ @a.get("/health")
31+ def health():
32+ return {"status": "ok"}
33+
34+ http.install(a)
35+ assert spa.install(a, dist) is True
36+ with TestClient(a) as c:
37+ yield c
38+
39+
40+def test_root_and_known_routes_serve_shell_200(spa_client):
41+ for path in ("/", "/playground", "/signin", "/dashboard", "/dashboard/keys", "/admin/users", "/integrations/mcp", "/pricing"):
42+ r = spa_client.get(path)
43+ assert r.status_code == 200, path
44+ assert "text/html" in r.headers["content-type"] and "<div id=root>" in r.text
45+ assert r.headers["Cache-Control"] == "no-cache"
46+
47+
48+def test_prerendered_shells_served(spa_client):
49+ r = spa_client.get("/docs")
50+ assert r.status_code == 200 and "prerendered docs" in r.text
51+ r = spa_client.get("/docs/errors") # no prerender → shell, still a known route
52+ assert r.status_code == 200 and "<div id=root>" in r.text
53+ r = spa_client.get("/limits")
54+ assert r.status_code == 200 and "prerendered limits" in r.text
55+
56+
57+def test_unknown_route_is_404_with_shell(spa_client):
58+ r = spa_client.get("/this-page-does-not-exist")
59+ assert r.status_code == 404
60+ assert "<div id=root>" in r.text and "text/html" in r.headers["content-type"]
61+ r = spa_client.get("/dashboardx")
62+ assert r.status_code == 404
63+
64+
65+def test_real_files_served(spa_client):
66+ r = spa_client.get("/favicon.svg")
67+ assert r.status_code == 200 and "svg" in r.headers["content-type"]
68+
69+
70+def test_assets_immutable_and_gzipped(spa_client):
71+ r = spa_client.get("/assets/index-abc123.js", headers={"Accept-Encoding": "gzip"})
72+ assert r.status_code == 200
73+ assert r.headers["Cache-Control"] == "public, max-age=31536000, immutable"
74+ assert r.headers.get("Content-Encoding") == "gzip"
75+ assert r.headers["X-Frame-Options"] == "DENY"
76+ r = spa_client.get("/assets/missing-000.js")
77+ assert r.status_code == 404
78+ assert "Cache-Control" not in r.headers or "immutable" not in r.headers["Cache-Control"]
79+
80+
81+def test_html_gets_csp_not_frame_options(spa_client):
82+ r = spa_client.get("/")
83+ csp = r.headers["Content-Security-Policy"]
84+ assert "default-src 'self'" in csp and "frame-ancestors 'none'" in csp and "cdn.jsdelivr.net" not in csp
85+ assert "X-Frame-Options" not in r.headers
86+ assert r.headers["Strict-Transport-Security"].startswith("max-age=")
87+
88+
89+def test_api_paths_never_get_the_shell(spa_client):
90+ r = spa_client.get("/v1/does-not-exist")
91+ assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
92+ assert r.json()["error"]["code"] == "NOT_FOUND"
93+ r = spa_client.get("/v1/ping")
94+ assert r.status_code == 200 and r.json() == {"ok": True}
95+ r = spa_client.get("/openapi.json")
96+ assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
97+ r = spa_client.get("/swagger")
98+ assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
99+
100+
101+def test_login_redirect(spa_client):
102+ r = spa_client.get("/login", follow_redirects=False)
103+ assert r.status_code == 301 and r.headers["Location"] == "/signin"
104+
105+
106+def test_path_traversal_blocked(spa_client):
107+ r = spa_client.get("/../../etc/passwd")
108+ assert r.status_code in (404, 200)
109+ assert "root:" not in r.text
110
Menu

Theme
↑↓navigate ↵open escclose /⌘Kanywhere