SPB Git forge
spboucher.ai ↗

spb/hfmarketdata

Public

Open high-frequency market data platform — FirstRate full-history downloader, DuckDB/Parquet lake, open REST API and React docs platform (www.hfmarketdata.io)

127commits 1branches 0releases
24.7 MBsize
maindefault branch
11 days agolast push
JavaScript 53.7% Python 38.3% CSS 4.6% TypeScript 3.1%

api: v1 robuste — 400 au lieu de 500, tickers partiels, limites de palier, gzip, X-Request-ID, /health étendu, /swagger

- bars/bars_multi/snapshot/options : dates parsées en Python et liées typées (start=bad → 400 INVALID_PARAMETER,
  end < start → 400), > 50 tickers → 400 TOO_MANY_TICKERS, ticker absent dans une requête multi → 200 + en-tête
  X-Missing-Tickers (404 seulement si aucun n'existe), `limit` borné par request.state.max_rows et les plafonds existants,
  LIMIT poussé dans la requête (plus de matérialisation intégrale puis .limit())
- bars_multi/snapshot via core.duck.run_topn (un seul scan filtré) ; suppression des doublons _tls/_cache de main.py
  au profit de core.duck.con()/cached()
- GZipMiddleware(minimum_size=1024) ; core.http.install (X-Request-ID, access log JSON, en-têtes de sécurité) ;
  Cache-Control: public, max-age=86400 sur les fenêtres de barres closes avant aujourd'hui
- /openapi.json : route dédiée (ETag + Cache-Control, 304) au lieu du middleware BaseHTTP ; Swagger déplacé sur
  /swagger, ReDoc sur /redoc — /docs revient au site React ; fallback SPA via core.spa (404 pour chemins inconnus)
- /health : {status, checks, version} 200/503, hors quota ; /v1/status hors quota lignes
- lifespan branché (garde secrets prod, pool de threads borné) ; OpenAPI : X-Request-ID, X-Missing-Tickers, TOO_MANY_TICKERS

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 18 days ago (Sep 6, 2026) parent b2e3721

4 changed files +685 −204

modified hfmarketdata/api/main.py +255 −203
@@ -24,23 +24,37 @@ Contact : contact@spboucher.ai
24 24
25 25 from __future__ import annotations
26 26
27 +import hashlib
27 28 import os
28 −import threading
29 −import time
29 +from datetime import datetime
30 30 from pathlib import Path
31 +from zoneinfo import ZoneInfo
31 32
32 −import duckdb
33 −from fastapi import FastAPI, HTTPException, Query
34 −from fastapi.middleware.cors import CORSMiddleware
35 −from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, Response
36 −from fastapi.staticfiles import StaticFiles
37 −
33 +import pandas as pd
38 34 from core import errors as _errors
39 −from core.config import settings as _settings
35 +from core import health as _health
36 +from core import http as _http
37 +from core import spa as _spa
38 +from core.duck import cached, con, run_topn
39 +from core.errors import ApiError
40 +from core.lifespan import lifespan
41 +from core.params import (
42 + MAX_TICKERS,
43 + bound_limit,
44 + parse_date,
45 + parse_datetime,
46 + parse_range,
47 + split_tickers,
48 +)
49 +from fastapi import FastAPI, Query, Request
50 +from fastapi.middleware.cors import CORSMiddleware
51 +from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
52 +from fastapi.responses import JSONResponse, PlainTextResponse, Response
53 +from starlette.middleware.gzip import GZipMiddleware
40 54
41 55 __author__ = "Simon-Pierre Boucher"
42 56 __contact__ = "contact@spboucher.ai"
43 −__version__ = "2.0.0"
57 +__version__ = "2.1.0"
44 58
45 59 DATA_ROOT = Path(os.environ.get("HFMD_DATA_ROOT", "/Volumes/ssd/firstratedata"))
46 60 PARQUET = DATA_ROOT / "parquet"
@@ -56,6 +70,18 @@ DEFAULT_ADJUSTMENT = {
56 70 }
57 71 MAX_LIMIT_JSON = 50_000
58 72 MAX_LIMIT_CSV = 2_000_000
73 +ET = ZoneInfo("America/New_York")
74 +HISTORY_CACHE = "public, max-age=86400" # bars whose window ends before today never change
75 +
76 +_LIMIT_DOC = ("Max rows. Silently capped at 50 000 (JSON) / 2 000 000 (CSV) and at your tier's "
77 + "`max_rows_per_request` (see /v1/limits).")
78 +_RANGE_DOC = ("ISO 8601 date or datetime (`2024-06-03`, `2024-06-03 10:35:00`, `2024-06-03T10:35:00Z`). "
79 + "Lake timestamps are US/Eastern (intraday) and naive; a bare date means midnight. "
80 + "Malformed values → 400 INVALID_PARAMETER.")
81 +_MISSING_DOC = ("Tickers that exist are returned even when others are unknown: the response is 200 with the "
82 + "header `X-Missing-Tickers: A,B` listing the ones not found; 404 only when none exists.")
83 +_MISSING_HEADER = {"X-Missing-Tickers": {"schema": {"type": "string"},
84 + "description": "Comma-separated requested tickers that do not exist for this asset / timeframe / adjustment (only when some are missing)."}}
59 85
60 86 app = FastAPI(
61 87 title="HF Market Data API",
@@ -68,48 +94,27 @@ app = FastAPI(
68 94 version=__version__,
69 95 contact={"name": __author__, "email": __contact__,
70 96 "url": "https://www.hfmarketdata.io"},
97 + # `/docs` belongs to the React documentation site; Swagger UI lives at /swagger, ReDoc at /redoc.
98 + # The OpenAPI document is served by a dedicated route below (ETag + Cache-Control).
99 + openapi_url=None, docs_url=None, redoc_url=None,
100 + lifespan=lifespan,
71 101 )
72 102 app.add_middleware(
73 103 CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
74 − expose_headers=["X-Row-Count", "X-RateLimit-Limit-Requests", "X-RateLimit-Remaining-Requests",
104 + expose_headers=["X-Row-Count", "X-Request-ID", "X-Missing-Tickers", "ETag",
105 + "X-RateLimit-Limit-Requests", "X-RateLimit-Remaining-Requests",
75 106 "X-RateLimit-Limit-Rows", "X-RateLimit-Remaining-Rows", "X-RateLimit-Reset", "Retry-After"],
76 107 )
108 +app.add_middleware(GZipMiddleware, minimum_size=1024)
77 109 _errors.install(app) # uniform {"error": {code, message, docs}} envelope (legacy "detail" kept)
78 110
79 −# ------------------------------------------------------------------------------
80 −# DuckDB — one connection per thread, read-only usage
81 −# ------------------------------------------------------------------------------
82 −
83 −_tls = threading.local()
84 −
85 −
86 −def db() -> duckdb.DuckDBPyConnection:
87 − if not hasattr(_tls, "con"):
88 − _tls.con = duckdb.connect()
89 − _tls.con.execute("SET threads TO 4")
90 − _tls.con.execute("SET enable_object_cache=true")
91 − return _tls.con
92 −
93 111
94 112 # ------------------------------------------------------------------------------
95 −# File resolution with a small TTL cache (directory scans are the slow part)
113 +# DuckDB: `core.duck.con()` = thread-local cursor on the single per-process database;
114 +# `core.duck.cached()` = TTL/LRU cache for directory scans.
96 115 # ------------------------------------------------------------------------------
97 116
98 −_cache: dict[str, tuple[float, object]] = {}
99 −_cache_lock = threading.Lock()
100 −CACHE_TTL = 300 # seconds
101 −
102 −
103 −def cached(key: str, builder):
104 − now = time.time()
105 − with _cache_lock:
106 − hit = _cache.get(key)
107 − if hit and now - hit[0] < CACHE_TTL:
108 − return hit[1]
109 − value = builder()
110 − with _cache_lock:
111 − _cache[key] = (now, value)
112 − return value
117 +db = con # legacy alias
113 118
114 119
115 120 def bar_dir(asset: str, timeframe: str, adjustment: str) -> Path:
@@ -165,38 +170,61 @@ def options_files(ticker: str) -> list[str]:
165 170
166 171 def _validate(asset: str, timeframe: str, adjustment: str | None) -> str:
167 172 if asset not in BAR_TYPES:
168 − raise HTTPException(404, f"Unknown asset type '{asset}'. "
169 − f"Available: {', '.join(BAR_TYPES)}")
173 + raise ApiError(404, "ASSET_NOT_FOUND", f"Unknown asset type '{asset}'. "
174 + f"Available: {', '.join(BAR_TYPES)}")
170 175 if timeframe not in TIMEFRAMES:
171 − raise HTTPException(400, f"Unknown timeframe '{timeframe}'. "
172 − f"Available: {', '.join(TIMEFRAMES)}")
176 + raise ApiError(400, "INVALID_PARAMETER", f"Unknown timeframe '{timeframe}'. "
177 + f"Available: {', '.join(TIMEFRAMES)}")
173 178 adj = adjustment or DEFAULT_ADJUSTMENT[asset]
174 179 available = list_adjustments(asset, timeframe)
175 180 if available and adj not in available:
176 − raise HTTPException(400, f"Adjustment '{adj}' not available for "
177 − f"{asset}/{timeframe}. Available: {', '.join(available)}")
181 + raise ApiError(400, "INVALID_PARAMETER", f"Adjustment '{adj}' not available for "
182 + f"{asset}/{timeframe}. Available: {', '.join(available)}")
178 183 return adj
179 184
180 185
181 −def _rows_to_response(rel, limit: int, fmt: str, order_col: str):
186 +def _resolve_tickers(idx: dict[str, str], names: list[str], where: str) -> tuple[list[str], list[str], dict[str, str]]:
187 + """(found, missing, extra headers): 404 only when NO requested ticker exists."""
188 + found = [t for t in names if t in idx]
189 + missing = [t for t in names if t not in idx]
190 + if not found:
191 + raise ApiError(404, "TICKER_NOT_FOUND", f"Not found in {where}: {', '.join(missing)}",
192 + details={"missing": missing})
193 + return found, missing, ({"X-Missing-Tickers": ",".join(missing)} if missing else {})
194 +
195 +
196 +def _range_conditions(lo: datetime | None, hi: datetime | None) -> tuple[str, list]:
197 + conds, params = [], []
198 + if lo is not None:
199 + conds.append("datetime >= ?"); params.append(lo)
200 + if hi is not None:
201 + conds.append("datetime <= ?"); params.append(hi)
202 + return " AND ".join(conds), params
203 +
204 +
205 +def _history_headers(upper: datetime | None) -> dict[str, str]:
206 + """Immutable history: a window that ends strictly before today (Eastern) can be cached for a day."""
207 + if upper is not None and upper.date() < datetime.now(ET).date():
208 + return {"Cache-Control": HISTORY_CACHE}
209 + return {}
210 +
211 +
212 +def _frame_response(df: pd.DataFrame, fmt: str, headers: dict[str, str] | None = None) -> Response:
213 + """v1 envelope: CSV text or `{"count", "data"}` JSON, always with `X-Row-Count`."""
214 + hdrs = {"X-Row-Count": str(len(df)), **(headers or {})}
182 215 if fmt == "csv":
183 − limit = min(limit, MAX_LIMIT_CSV)
184 − rel = rel.limit(limit)
185 − df = rel.df()
186 − return PlainTextResponse(df.to_csv(index=False), media_type="text/csv",
187 − headers={"X-Row-Count": str(len(df))})
188 − limit = min(limit, MAX_LIMIT_JSON)
189 − rel = rel.limit(limit)
190 − df = rel.df()
216 + return PlainTextResponse(df.to_csv(index=False), media_type="text/csv", headers=hdrs)
191 217 for col in df.columns:
192 218 if str(df[col].dtype).startswith("datetime"):
193 219 df[col] = df[col].astype(str)
194 − # NaN / ±inf (Greeks manquants dans les vieux trimestres) ne sont pas
195 − # sérialisables en JSON strict → null
220 + # NaN / ±inf (missing Greeks in old quarters) are not strict JSON → null
196 221 df = df.replace([float("inf"), float("-inf")], None)
197 222 df = df.astype(object).where(df.notna(), None)
198 − return JSONResponse({"count": len(df), "data": df.to_dict(orient="records")},
199 − headers={"X-Row-Count": str(len(df))})
223 + return JSONResponse({"count": len(df), "data": df.to_dict(orient="records")}, headers=hdrs)
224 +
225 +
226 +def _hard_max(fmt: str) -> int:
227 + return MAX_LIMIT_CSV if fmt == "csv" else MAX_LIMIT_JSON
200 228
201 229
202 230 # ------------------------------------------------------------------------------
@@ -204,16 +232,27 @@ def _rows_to_response(rel, limit: int, fmt: str, order_col: str):
204 232 # ------------------------------------------------------------------------------
205 233
206 234
207 −@app.get("/health", tags=["meta"])
208 −def health():
209 − return {"status": "ok", "service": "hfmarketdata-api", "version": __version__,
210 − "data_root_present": PARQUET.is_dir()}
235 +@app.get("/health", tags=["meta"], summary="Service health",
236 + description="Liveness + dependency checks: Parquet lake, SQLite state DB, Redis (300 ms timeout) and a witness "
237 + "`read_parquet` (cached 60 s). `status` is `ok` (HTTP 200) or `degraded` (HTTP 503). Never counted "
238 + "against any quota.",
239 + responses={200: {"description": "Healthy", "content": {"application/json": {"example": {
240 + "status": "ok", "service": "hfmarketdata-api", "version": __version__, "data_root_present": True,
241 + "checks": {"parquet": {"ok": True, "path": "/…/parquet"}, "sqlite": {"ok": True, "path": "/…/hfmd.db"},
242 + "redis": {"ok": True}, "duckdb": {"ok": True, "file": "AAPL_1day.parquet", "rows": 4200, "ms": 3.1}}}}}},
243 + 503: {"description": "Degraded (same body, `status: degraded`)"}})
244 +def health(request: Request):
245 + request.state.quota_exempt = True
246 + body, status = _health.report(__version__)
247 + return JSONResponse(body, status_code=status, headers={"Cache-Control": "no-store"})
211 248
212 249
213 250 @app.get("/v1/status", tags=["meta"])
214 −def status():
251 +def status(request: Request):
215 252 """Dataset inventory: every asset type, timeframe and adjustment with
216 − the number of instruments currently available."""
253 + the number of instruments currently available. Not charged against the rows quota."""
254 + request.state.quota_exempt = True
255 +
217 256 def build():
218 257 out = {}
219 258 for asset in BAR_TYPES:
@@ -265,102 +304,96 @@ def tickers(asset: str,
265 304 # ------------------------------------------------------------------------------
266 305
267 306
268 −@app.get("/v1/bars/{asset}/{ticker}", tags=["bars"])
269 −def bars(asset: str, ticker: str,
307 +@app.get("/v1/bars/{asset}/{ticker}", tags=["bars"],
308 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
309 +def bars(request: Request, asset: str, ticker: str,
270 310 timeframe: str = Query("1day", description="1min|5min|30min|1hour|1day"),
271 311 adjustment: str | None = Query(None, description="Default depends on asset type"),
272 − start: str | None = Query(None, description="ISO date/datetime lower bound"),
273 − end: str | None = Query(None, description="ISO date/datetime upper bound"),
312 + start: str | None = Query(None, description="Lower bound (inclusive). " + _RANGE_DOC),
313 + end: str | None = Query(None, description="Upper bound (inclusive), must be ≥ `start`. " + _RANGE_DOC),
274 314 order: str = Query("asc", pattern="^(asc|desc)$"),
275 − limit: int = Query(5_000, ge=1),
315 + limit: int = Query(5_000, ge=1, description=_LIMIT_DOC),
276 316 format: str = Query("json", pattern="^(json|csv)$")):
277 317 """OHLCV bars for one instrument. Daily futures bars also carry open
278 − interest. `format=csv` allows bulk extraction (up to 2M rows)."""
318 + interest. `format=csv` allows bulk extraction (up to 2M rows).
319 +
320 + Responses whose `end` is strictly before today are immutable history and carry
321 + `Cache-Control: public, max-age=86400`."""
279 322 adj = _validate(asset, timeframe, adjustment)
280 323 idx = ticker_index(asset, timeframe, adj)
281 324 path = idx.get(ticker.upper())
282 325 if not path:
283 − raise HTTPException(404, f"Ticker '{ticker.upper()}' not found in "
284 − f"{asset}/{timeframe}/{adj}")
285 − con = db()
286 − conds, params = [], []
287 − if start:
288 − conds.append("datetime >= ?"); params.append(start)
289 − if end:
290 − conds.append("datetime <= ?"); params.append(end)
291 − where = ("WHERE " + " AND ".join(conds)) if conds else ""
292 − rel = con.sql(
293 − f"SELECT * FROM read_parquet(?) {where} ORDER BY datetime {order.upper()}",
294 − params=[path, *params])
295 − return _rows_to_response(rel, limit, format, "datetime")
296 −
297 −
298 −@app.get("/v1/bars/{asset}", tags=["bars"])
299 −def bars_multi(asset: str,
300 − tickers: str = Query(..., description="Comma-separated list, e.g. AAPL,MSFT,TSLA (max 50)"),
326 + raise ApiError(404, "TICKER_NOT_FOUND", f"Ticker '{ticker.upper()}' not found in "
327 + f"{asset}/{timeframe}/{adj}")
328 + lo, hi = parse_range(start, end)
329 + where_sql, params = _range_conditions(lo, hi)
330 + where = f"WHERE {where_sql}" if where_sql else ""
331 + lim = bound_limit(limit, _hard_max(format), request)
332 + df = con().execute(
333 + f"SELECT * FROM read_parquet(?) {where} ORDER BY datetime {order.upper()} LIMIT ?",
334 + [path, *params, lim]).df()
335 + return _frame_response(df, format, _history_headers(hi))
336 +
337 +
338 +@app.get("/v1/bars/{asset}", tags=["bars"],
339 + responses={200: {"description": "Bars of the found tickers", "headers": _MISSING_HEADER}},
340 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "TOO_MANY_TICKERS", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
341 +def bars_multi(request: Request, asset: str,
342 + tickers: str = Query(..., description=f"Comma-separated list, e.g. AAPL,MSFT,TSLA (max {MAX_TICKERS}; "
343 + "more → 400 TOO_MANY_TICKERS). " + _MISSING_DOC),
301 344 timeframe: str = Query("1day"),
302 345 adjustment: str | None = Query(None),
303 − start: str | None = Query(None, description="ISO date/datetime lower bound"),
304 − end: str | None = Query(None, description="ISO date/datetime upper bound"),
346 + start: str | None = Query(None, description="Lower bound (inclusive). " + _RANGE_DOC),
347 + end: str | None = Query(None, description="Upper bound (inclusive), must be ≥ `start`. " + _RANGE_DOC),
305 348 order: str = Query("asc", pattern="^(asc|desc)$"),
306 − limit: int = Query(5_000, ge=1, description="Max rows PER TICKER"),
349 + limit: int = Query(5_000, ge=1, description="Max rows PER TICKER. The total (limit × tickers) is also "
350 + "capped by the endpoint ceiling and your tier's `max_rows_per_request`."),
307 351 format: str = Query("json", pattern="^(json|csv)$")):
308 352 """Bars for SEVERAL instruments in one call. Combine with `start`/`end`
309 353 to slice any precise window — e.g. 4 hours of 1-minute bars across a
310 − whole watchlist. The row limit applies per ticker."""
354 + whole watchlist. The row limit applies per ticker.
355 +
356 + Unknown tickers do not fail the whole call: they are listed in the `X-Missing-Tickers`
357 + response header (404 only when none of them exists). Windows ending before today carry
358 + `Cache-Control: public, max-age=86400`."""
311 359 adj = _validate(asset, timeframe, adjustment)
312 360 idx = ticker_index(asset, timeframe, adj)
313 − names = [t.strip().upper() for t in tickers.split(",") if t.strip()][:50]
314 − if not names:
315 − raise HTTPException(400, "No tickers given")
316 − missing = [t for t in names if t not in idx]
317 − if missing:
318 − raise HTTPException(404, f"Not found in {asset}/{timeframe}/{adj}: "
319 − f"{', '.join(missing)}")
320 − paths = [idx[t] for t in names]
321 − con = db()
322 − conds, params = [], []
323 − if start:
324 − conds.append("datetime >= ?"); params.append(start)
325 − if end:
326 − conds.append("datetime <= ?"); params.append(end)
327 − where = ("WHERE " + " AND ".join(conds)) if conds else ""
328 − per_ticker = min(limit, MAX_LIMIT_CSV if format == "csv" else MAX_LIMIT_JSON)
329 − rel = con.sql(
330 − f"SELECT * FROM read_parquet(?) {where} "
331 − f"QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY datetime "
332 − f"{order.upper()}) <= ? ORDER BY ticker, datetime {order.upper()}",
333 − params=[paths, *params, per_ticker])
334 − return _rows_to_response(rel, per_ticker * len(names), format, "datetime")
335 −
336 −
337 −@app.get("/v1/snapshot/{asset}", tags=["bars"])
338 −def snapshot(asset: str,
339 − tickers: str = Query(..., description="Comma-separated list, e.g. AAPL,MSFT,TSLA (max 50)"),
340 − at: str = Query(..., description="Precise moment, e.g. 2024-06-03 10:35:00"),
361 + names = split_tickers(tickers)
362 + found, _missing, extra = _resolve_tickers(idx, names, f"{asset}/{timeframe}/{adj}")
363 + lo, hi = parse_range(start, end)
364 + where_sql, params = _range_conditions(lo, hi)
365 + hard = _hard_max(format)
366 + per_ticker = bound_limit(limit, hard, request)
367 + total = bound_limit(per_ticker * len(found), hard, request)
368 + paths = [idx[t] for t in found]
369 + df = run_topn(con(), paths, where_sql, params, "ticker", f"datetime {order.upper()}", per_ticker,
370 + order_by=f"ticker, datetime {order.upper()}", limit=total)
371 + return _frame_response(df, format, {**extra, **_history_headers(hi)})
372 +
373 +
374 +@app.get("/v1/snapshot/{asset}", tags=["bars"],
375 + responses={200: {"description": "Last bar at or before `at` for each found ticker", "headers": _MISSING_HEADER}},
376 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "TOO_MANY_TICKERS", "ASSET_NOT_FOUND", "TICKER_NOT_FOUND"]})
377 +def snapshot(request: Request, asset: str,
378 + tickers: str = Query(..., description=f"Comma-separated list, e.g. AAPL,MSFT,TSLA (max {MAX_TICKERS}). " + _MISSING_DOC),
379 + at: str = Query(..., description="Precise moment, e.g. 2024-06-03 10:35:00. " + _RANGE_DOC),
341 380 timeframe: str = Query("1min"),
342 381 adjustment: str | None = Query(None),
343 382 format: str = Query("json", pattern="^(json|csv)$")):
344 383 """Cross-sectional snapshot: for each requested instrument, the last bar
345 384 at or before the given moment — the state of a whole watchlist at one
346 − precise point in time."""
385 + precise point in time. Unknown tickers are reported in `X-Missing-Tickers`."""
347 386 adj = _validate(asset, timeframe, adjustment)
348 387 idx = ticker_index(asset, timeframe, adj)
349 − names = [t.strip().upper() for t in tickers.split(",") if t.strip()][:50]
350 − if not names:
351 − raise HTTPException(400, "No tickers given")
352 − missing = [t for t in names if t not in idx]
353 − if missing:
354 − raise HTTPException(404, f"Not found in {asset}/{timeframe}/{adj}: "
355 − f"{', '.join(missing)}")
356 − paths = [idx[t] for t in names]
357 − con = db()
358 − rel = con.sql(
359 − "SELECT * FROM read_parquet(?) WHERE datetime <= ? "
360 − "QUALIFY row_number() OVER (PARTITION BY ticker ORDER BY datetime DESC) = 1 "
361 − "ORDER BY ticker",
362 − params=[paths, at])
363 − return _rows_to_response(rel, len(names), format, "datetime")
388 + names = split_tickers(tickers)
389 + found, _missing, extra = _resolve_tickers(idx, names, f"{asset}/{timeframe}/{adj}")
390 + at_dt = parse_datetime(at, "at")
391 + if at_dt is None:
392 + raise ApiError(400, "INVALID_PARAMETER", "at is required", details={"at": at})
393 + paths = [idx[t] for t in found]
394 + df = run_topn(con(), paths, "datetime <= ?", [at_dt], "ticker", "datetime DESC", 1,
395 + order_by="ticker", limit=len(found))
396 + return _frame_response(df, format, {**extra, **_history_headers(at_dt)})
364 397
365 398
366 399 # ------------------------------------------------------------------------------
@@ -381,10 +414,10 @@ def opt_tickers(quarter: str | None = Query(None, description="e.g. 2024_q4; def
381 414 """Underlyings available in a given options quarter."""
382 415 quarters = options_quarters()
383 416 if not quarters:
384 − raise HTTPException(503, "Options dataset not yet available")
417 + raise ApiError(503, "SERVICE_UNAVAILABLE", "Options dataset not yet available")
385 418 q = quarter or quarters[-1]
386 419 if q not in quarters:
387 − raise HTTPException(404, f"Unknown quarter '{q}'")
420 + raise ApiError(404, "NOT_FOUND", f"Unknown quarter '{q}'")
388 421
389 422 def build():
390 423 return sorted({p.stem.split("_")[0].upper()
@@ -396,32 +429,39 @@ def opt_tickers(quarter: str | None = Query(None, description="e.g. 2024_q4; def
396 429 return {"quarter": q, "count": len(names[:limit]), "tickers": names[:limit]}
397 430
398 431
399 −@app.get("/v1/options/chain/{ticker}", tags=["options"])
400 −def opt_chain(ticker: str,
432 +def _no_options(ticker: str) -> ApiError:
433 + return ApiError(404, "OPTIONS_UNAVAILABLE", f"No options data for '{ticker.upper()}'")
434 +
435 +
436 +@app.get("/v1/options/chain/{ticker}", tags=["options"],
437 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
438 +def opt_chain(request: Request, ticker: str,
401 439 trade_date: str | None = Query(None, description="yyyy-mm-dd; default latest available"),
402 − expiry: str | None = Query(None, description="Filter on expiry date"),
440 + expiry: str | None = Query(None, description="Filter on expiry date (yyyy-mm-dd)"),
403 441 call_put: str | None = Query(None, pattern="^(c|p)$"),
404 442 strike_min: float | None = None,
405 443 strike_max: float | None = None,
406 444 min_volume: float | None = None,
407 − limit: int = Query(20_000, ge=1),
445 + limit: int = Query(20_000, ge=1, description=_LIMIT_DOC),
408 446 format: str = Query("json", pattern="^(json|csv)$")):
409 447 """Full end-of-day option chain for an underlying: quotes, bid/ask implied
410 448 volatility, open interest, volume and Greeks (delta, gamma, vega, theta, rho)."""
411 449 files = options_files(ticker)
412 450 if not files:
413 − raise HTTPException(404, f"No options data for '{ticker.upper()}'")
414 − con = db()
415 − if trade_date is None:
416 − trade_date = con.execute(
417 − "SELECT max(trade_date) FROM read_parquet(?)", [files[-1]]
418 − ).fetchone()[0]
451 + raise _no_options(ticker)
452 + c = con()
453 + td = parse_date(trade_date, "trade_date")
454 + if td is None:
455 + latest = c.execute("SELECT max(trade_date) FROM read_parquet(?)", [files[-1]]).fetchone()[0]
456 + td_s = str(latest)
419 457 files_q = [files[-1]]
420 458 else:
459 + td_s = td.isoformat()
421 460 files_q = files
422 − conds, params = ["trade_date = ?"], [str(trade_date)]
423 − if expiry:
424 − conds.append("expiry = ?"); params.append(expiry)
461 + conds, params = ["trade_date = ?"], [td_s]
462 + exp = parse_date(expiry, "expiry")
463 + if exp is not None:
464 + conds.append("expiry = ?"); params.append(exp.isoformat())
425 465 if call_put:
426 466 conds.append("call_put = ?"); params.append(call_put)
427 467 if strike_min is not None:
@@ -430,49 +470,54 @@ def opt_chain(ticker: str,
430 470 conds.append("strike <= ?"); params.append(strike_max)
431 471 if min_volume is not None:
432 472 conds.append("volume >= ?"); params.append(min_volume)
433 − rel = con.sql(
473 + lim = bound_limit(limit, _hard_max(format), request)
474 + df = c.execute(
434 475 f"SELECT * FROM read_parquet(?) WHERE {' AND '.join(conds)} "
435 − f"ORDER BY expiry, strike, call_put",
436 − params=[files_q, *params])
437 − return _rows_to_response(rel, limit, format, "expiry")
476 + f"ORDER BY expiry, strike, call_put LIMIT ?",
477 + [files_q, *params, lim]).df()
478 + return _frame_response(df, format)
438 479
439 480
440 −@app.get("/v1/options/expirations/{ticker}", tags=["options"])
481 +@app.get("/v1/options/expirations/{ticker}", tags=["options"],
482 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
441 483 def opt_expirations(ticker: str,
442 484 trade_date: str | None = Query(None, description="yyyy-mm-dd")):
443 485 """Available expiry dates (optionally as of one trade date)."""
444 486 files = options_files(ticker)
445 487 if not files:
446 − raise HTTPException(404, f"No options data for '{ticker.upper()}'")
447 − con = db()
448 − if trade_date:
449 − rows = con.execute(
488 + raise _no_options(ticker)
489 + c = con()
490 + td = parse_date(trade_date, "trade_date")
491 + if td is not None:
492 + rows = c.execute(
450 493 "SELECT DISTINCT expiry FROM read_parquet(?) WHERE trade_date = ? ORDER BY expiry",
451 − [files, trade_date]).fetchall()
494 + [files, td.isoformat()]).fetchall()
452 495 else:
453 − rows = con.execute(
496 + rows = c.execute(
454 497 "SELECT DISTINCT expiry FROM read_parquet(?) ORDER BY expiry",
455 498 [files[-1]]).fetchall()
456 499 return {"ticker": ticker.upper(), "expirations": [str(r[0]) for r in rows]}
457 500
458 501
459 −@app.get("/v1/options/history/{ticker}", tags=["options"])
460 −def opt_history(ticker: str,
502 +@app.get("/v1/options/history/{ticker}", tags=["options"],
503 + openapi_extra={"x-errors": ["INVALID_PARAMETER", "OPTIONS_UNAVAILABLE"]})
504 +def opt_history(request: Request, ticker: str,
461 505 strike: float = Query(...),
462 506 expiry: str = Query(..., description="yyyy-mm-dd"),
463 507 call_put: str = Query(..., pattern="^(c|p)$"),
464 − limit: int = Query(5_000, ge=1),
508 + limit: int = Query(5_000, ge=1, description=_LIMIT_DOC),
465 509 format: str = Query("json", pattern="^(json|csv)$")):
466 510 """Daily time series for one specific contract across its whole life."""
467 511 files = options_files(ticker)
468 512 if not files:
469 − raise HTTPException(404, f"No options data for '{ticker.upper()}'")
470 − con = db()
471 − rel = con.sql(
513 + raise _no_options(ticker)
514 + exp = parse_date(expiry, "expiry")
515 + lim = bound_limit(limit, _hard_max(format), request)
516 + df = con().execute(
472 517 "SELECT * FROM read_parquet(?) WHERE strike = ? AND expiry = ? AND call_put = ? "
473 − "ORDER BY trade_date",
474 − params=[files, strike, expiry, call_put])
475 − return _rows_to_response(rel, limit, format, "trade_date")
518 + "ORDER BY trade_date LIMIT ?",
519 + [files, strike, exp.isoformat(), call_put, lim]).df()
520 + return _frame_response(df, format)
476 521
477 522
478 523 # ------------------------------------------------------------------------------
@@ -498,44 +543,51 @@ for _mod in V2_MODULES:
498 543 except Exception as e: # pragma: no cover
499 544 _log.exception("module %s failed to load: %s", _mod, e)
500 545
546 +# Request id + JSON access log + security headers: outermost layer, mounted after every other middleware.
547 +_http.install(app)
548 +
549 +
550 +# ------------------------------------------------------------------------------
551 +# OpenAPI document (ETag + Cache-Control) and the interactive consoles
552 +# ------------------------------------------------------------------------------
553 +
554 +_OPENAPI_CACHE = "public, max-age=300"
555 +
556 +
557 +def _openapi_body() -> tuple[bytes, str]:
558 + def build():
559 + import json
560 + body = json.dumps(app.openapi(), separators=(",", ":"), ensure_ascii=False).encode()
561 + return body, '"' + hashlib.sha256(body).hexdigest()[:32] + '"'
562 + return cached("openapi|json", build, ttl=3600)
501 563
502 −# ETag on the OpenAPI document so the docs' If-None-Match revalidation is cheap
503 −import hashlib as _hashlib
504 564
505 −@app.middleware("http")
506 −async def _openapi_etag(request, call_next):
507 − response = await call_next(request)
508 − if request.url.path == "/openapi.json" and response.status_code == 200:
509 − body = b"".join([chunk async for chunk in response.body_iterator])
510 − etag = '"' + _hashlib.sha256(body).hexdigest()[:32] + '"'
511 − if request.headers.get("if-none-match") == etag:
512 − return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "public, max-age=300"})
513 − return Response(body, status_code=200, media_type="application/json",
514 − headers={"ETag": etag, "Cache-Control": "public, max-age=300"})
515 − return response
565 +@app.get("/openapi.json", include_in_schema=False)
566 +def openapi_json(request: Request):
567 + body, etag = _openapi_body()
568 + if etag in [t.strip() for t in (request.headers.get("if-none-match") or "").split(",")]:
569 + return Response(status_code=304, headers={"ETag": etag, "Cache-Control": _OPENAPI_CACHE})
570 + return Response(body, media_type="application/json", headers={"ETag": etag, "Cache-Control": _OPENAPI_CACHE})
571 +
572 +
573 +@app.get("/swagger", include_in_schema=False)
574 +def swagger_ui():
575 + return get_swagger_ui_html(openapi_url="/openapi.json", title=f"{app.title} — Swagger UI",
576 + swagger_favicon_url="/favicon.svg")
577 +
578 +
579 +@app.get("/redoc", include_in_schema=False)
580 +def redoc_ui():
581 + return get_redoc_html(openapi_url="/openapi.json", title=f"{app.title} — ReDoc", redoc_favicon_url="/favicon.svg")
516 582
517 583
518 584 # ------------------------------------------------------------------------------
519 −# Static React platform (mounted last so it doesn't shadow the API) + SPA fallback
585 +# Static React platform (mounted last so it doesn't shadow the API) + SPA fallback — see core/spa.py
520 586 # ------------------------------------------------------------------------------
521 587
522 −if WEB_DIST.is_dir():
523 − app.mount("/assets", StaticFiles(directory=str(WEB_DIST / "assets")), name="assets")
524 −
525 − @app.get("/{path:path}", include_in_schema=False)
526 − def spa(path: str):
527 − if path.startswith(("v1/", "health", "openapi", "docs/", "redoc")) and not path.startswith("docs/"):
528 − raise HTTPException(404, "Not found")
529 − root = str(WEB_DIST.resolve())
530 − candidate = (WEB_DIST / path).resolve()
531 − if path and str(candidate).startswith(root):
532 − # exact file, prerendered shell (<route>.html or <route>/index.html), else the SPA entry
533 − for c in (candidate, candidate.with_suffix(".html") if candidate.suffix == "" else None, candidate / "index.html"):
534 − if c is not None and c.is_file():
535 − return FileResponse(c)
536 − return FileResponse(WEB_DIST / "index.html")
588 +_spa.install(app, WEB_DIST)
537 589
538 590
539 591 if __name__ == "__main__":
540 592 import uvicorn
541 − uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("HFMD_PORT", 8090)))
593 + uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("HFMD_PORT", 8090)), access_log=False)
modified hfmarketdata/api/openapi.py +2 −1
@@ -33,6 +33,7 @@ RATE_HEADERS = {
33 33 "X-RateLimit-Remaining-Rows": {"schema": {"type": "integer"}, "description": "Data rows left in the current window."},
34 34 "X-RateLimit-Reset": {"schema": {"type": "integer"}, "description": "Unix timestamp (seconds) when the window resets."},
35 35 "X-Row-Count": {"schema": {"type": "integer"}, "description": "Rows returned by this response."},
36 + "X-Request-ID": {"schema": {"type": "string"}, "description": "Correlation id of the request (echoed when you send one); quote it when reporting a problem."},
36 37 }
37 38
38 39 ERROR_SCHEMA = {
@@ -52,7 +53,7 @@ ERROR_SCHEMA = {
52 53 "docs": docs_link("CONTRACT_NOT_FOUND")}, "detail": "No data for contract ESZ19"},
53 54 }
54 55
55 −STATUS_FOR = {"INVALID_PARAMETER": 400, "VALIDATION_ERROR": 422, "NOT_FOUND": 404, "TICKER_NOT_FOUND": 404,
56 +STATUS_FOR = {"INVALID_PARAMETER": 400, "TOO_MANY_TICKERS": 400, "VALIDATION_ERROR": 422, "NOT_FOUND": 404, "TICKER_NOT_FOUND": 404,
56 57 "ASSET_NOT_FOUND": 404, "INVALID_CONTRACT_SYMBOL": 400, "CONTRACT_NOT_FOUND": 404, "ROOT_NOT_FOUND": 404,
57 58 "OPTIONS_UNAVAILABLE": 503, "INVALID_API_KEY": 401, "AUTH_REQUIRED": 401, "FORBIDDEN": 403,
58 59 "RATE_LIMIT_EXCEEDED": 429, "ROW_LIMIT_EXCEEDED": 400, "CONFLICT": 409, "INTERNAL_ERROR": 500,
added tests/test_api_hardening.py +319 −0
@@ -0,0 +1,319 @@
1 +"""Chantier api — strict v1 validation (400 instead of 500), partial multi-ticker results, tier-bounded limits,
2 +HTTP hygiene (gzip, X-Request-ID, security headers, caching), /health, Swagger moved to /swagger."""
3 +from __future__ import annotations
4 +
5 +import json
6 +import logging
7 +
8 +import pytest
9 +
10 +# ----------------------------------------------------------------------------------------------------- validation
11 +
12 +
13 +@pytest.mark.parametrize("url", [
14 + "/v1/bars/stock/AAPL?timeframe=1day&start=bad",
15 + "/v1/bars/stock/AAPL?timeframe=1day&end=2024-13-45",
16 + "/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01T25:00:00",
17 + "/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&start=nope",
18 + "/v1/snapshot/stock?tickers=AAPL&at=yesterday",
19 + "/v1/options/chain/AAPL?trade_date=2025-04-0x",
20 + "/v1/options/chain/AAPL?expiry=soon",
21 + "/v1/options/expirations/AAPL?trade_date=x",
22 + "/v1/options/history/AAPL?strike=200&expiry=x&call_put=c",
23 +])
24 +def test_invalid_dates_are_400(client, url):
25 + r = client.get(url)
26 + assert r.status_code == 400, r.text
27 + body = r.json()
28 + assert body["error"]["code"] == "INVALID_PARAMETER"
29 + assert body["detail"] # legacy field
30 + assert "Traceback" not in r.text
31 +
32 +
33 +def test_end_before_start_is_400(client):
34 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-06-10&end=2024-06-01")
35 + assert r.status_code == 400
36 + assert r.json()["error"]["code"] == "INVALID_PARAMETER"
37 + assert "start" in r.json()["error"]["details"]
38 +
39 +
40 +@pytest.mark.parametrize("start,end", [
41 + ("2024-06-03", "2024-06-04"),
42 + ("2024-06-03 09:30:00", "2024-06-03 09:34:59"),
43 + ("2024-06-03T09:30:00", "2024-06-03T09:34:59Z"),
44 + ("2024-06-03T09:30:00+00:00", None),
45 + ("2024-06-03T05:30:00-04:00", "2024-06-04"),
46 +])
47 +def test_accepted_date_forms(client, start, end):
48 + from urllib.parse import quote
49 + q = f"start={quote(start)}" + (f"&end={quote(end)}" if end else "")
50 + r = client.get(f"/v1/bars/stock/AAPL?timeframe=1day&{q}")
51 + assert r.status_code == 200, r.text
52 +
53 +
54 +def test_window_is_honoured_and_typed(client):
55 + r = client.get("/v1/bars/stock/AAPL?timeframe=1min&start=2025-06-30 09:30:00&end=2025-06-30 09:34:59&adjustment=UNADJUSTED")
56 + assert r.status_code == 200
57 + rows = r.json()["data"]
58 + assert len(rows) == 5
59 + assert rows[0]["datetime"].startswith("2025-06-30 09:30") and rows[-1]["datetime"].startswith("2025-06-30 09:34")
60 +
61 +
62 +def test_too_many_tickers_is_400(client):
63 + names = ",".join(f"T{i}" for i in range(51))
64 + r = client.get(f"/v1/bars/stock?tickers={names}&timeframe=1day")
65 + assert r.status_code == 400
66 + assert r.json()["error"]["code"] == "TOO_MANY_TICKERS"
67 + r = client.get(f"/v1/snapshot/stock?tickers={names}&at=2025-06-30 10:00:00")
68 + assert r.status_code == 400 and r.json()["error"]["code"] == "TOO_MANY_TICKERS"
69 +
70 +
71 +def test_fifty_tickers_ok_dedup(client):
72 + names = ",".join(["AAPL", "aapl", "MSFT"] + [f"T{i}" for i in range(47)]) # 49 distinct
73 + r = client.get(f"/v1/bars/stock?tickers={names}&timeframe=1day&limit=1")
74 + assert r.status_code == 200
75 + assert sorted({row["ticker"] for row in r.json()["data"]}) == ["AAPL", "MSFT"]
76 +
77 +
78 +def test_empty_tickers_is_400(client):
79 + r = client.get("/v1/bars/stock?tickers=,,&timeframe=1day")
80 + assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
81 +
82 +
83 +# -------------------------------------------------------------------------------------------- partial results
84 +
85 +
86 +def test_partially_missing_tickers_200_with_header(client):
87 + r = client.get("/v1/bars/stock?tickers=AAPL,NOPE,MSFT,ZZZ&timeframe=1day&limit=2")
88 + assert r.status_code == 200, r.text
89 + assert r.headers["X-Missing-Tickers"] == "NOPE,ZZZ"
90 + body = r.json()
91 + assert body["count"] == 4 and {row["ticker"] for row in body["data"]} == {"AAPL", "MSFT"}
92 + assert r.headers["X-Row-Count"] == "4"
93 +
94 +
95 +def test_all_missing_tickers_404(client):
96 + r = client.get("/v1/bars/stock?tickers=NOPE,ZZZ&timeframe=1day")
97 + assert r.status_code == 404
98 + assert r.json()["error"]["code"] == "TICKER_NOT_FOUND"
99 + assert r.json()["error"]["details"]["missing"] == ["NOPE", "ZZZ"]
100 +
101 +
102 +def test_no_missing_header_when_all_found(client):
103 + r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&limit=1")
104 + assert r.status_code == 200 and "X-Missing-Tickers" not in r.headers
105 +
106 +
107 +def test_snapshot_partial(client):
108 + r = client.get("/v1/snapshot/stock?tickers=AAPL,NOPE&at=2025-06-30 10:00:00&adjustment=UNADJUSTED")
109 + assert r.status_code == 200
110 + assert r.headers["X-Missing-Tickers"] == "NOPE"
111 + body = r.json()
112 + assert body["count"] == 1 and body["data"][0]["ticker"] == "AAPL"
113 + assert body["data"][0]["datetime"].startswith("2025-06-30 10:00")
114 +
115 +
116 +def test_multi_per_ticker_limit_and_order(client):
117 + r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&limit=3&order=desc")
118 + assert r.status_code == 200
119 + rows = r.json()["data"]
120 + assert len(rows) == 6
121 + aapl = [x["datetime"] for x in rows if x["ticker"] == "AAPL"]
122 + assert aapl == sorted(aapl, reverse=True) and aapl[0].startswith("2025-06-30")
123 +
124 +
125 +# ------------------------------------------------------------------------------------------------- limits
126 +
127 +
128 +def test_limit_bounded_by_tier_max_rows(client):
129 + """Keyless tier: 5 000 rows per request. 6 tickers × 1 950 one-minute bars would be 11 700 rows."""
130 + r = client.get("/v1/bars/stock?tickers=AAPL,MSFT,SMCP,SHAK,GOOG,GOOGL&timeframe=1min&limit=5000&adjustment=UNADJUSTED")
131 + assert r.status_code == 200
132 + assert r.headers["X-Row-Count"] == "5000" and r.json()["count"] == 5000
133 +
134 +
135 +def test_single_limit_bounded_by_tier_max_rows(client_hu, client):
136 + from core.params import bound_limit
137 +
138 + class _Req:
139 + class state:
140 + max_rows = 100
141 + assert bound_limit(5000, 50_000, _Req()) == 100
142 + assert bound_limit(50, 50_000, _Req()) == 50
143 +
144 + class _NoTier:
145 + class state:
146 + pass
147 + assert bound_limit(5_000_000, 50_000, _NoTier()) == 50_000
148 + assert bound_limit(10, 50_000, None) == 10
149 +
150 +
151 +# --------------------------------------------------------------------------------------------- HTTP hygiene
152 +
153 +
154 +def test_gzip_when_accepted(client):
155 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=500", headers={"Accept-Encoding": "gzip"})
156 + assert r.status_code == 200
157 + assert r.headers.get("Content-Encoding") == "gzip"
158 + assert r.json()["count"] == 500 # httpx decodes transparently
159 +
160 +
161 +def test_no_gzip_for_small_bodies(client):
162 + r = client.get("/health", headers={"Accept-Encoding": "gzip"})
163 + assert r.status_code in (200, 503)
164 + # /health is above 1 KiB only with many checks; the point is that gzip is negotiated, not forced
165 + r = client.get("/v1/options/quarters", headers={"Accept-Encoding": "gzip"})
166 + assert "Content-Encoding" not in r.headers
167 +
168 +
169 +def test_request_id_generated_and_echoed(client):
170 + r = client.get("/v1/options/quarters")
171 + rid = r.headers["X-Request-ID"]
172 + assert 8 <= len(rid) <= 64
173 + r2 = client.get("/v1/options/quarters", headers={"X-Request-ID": "trace-abc.123"})
174 + assert r2.headers["X-Request-ID"] == "trace-abc.123"
175 + r3 = client.get("/v1/options/quarters", headers={"X-Request-ID": "bad id with spaces"})
176 + assert r3.headers["X-Request-ID"] != "bad id with spaces"
177 +
178 +
179 +def test_request_id_on_errors_too(client):
180 + r = client.get("/v1/bars/stock/NOPE?timeframe=1day")
181 + assert r.status_code == 404 and r.headers.get("X-Request-ID")
182 +
183 +
184 +def test_access_log_line_is_json(client, caplog):
185 + with caplog.at_level(logging.INFO, logger="hfmarketdata.access"):
186 + client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=7", headers={"X-Request-ID": "log-test-1"})
187 + lines = [rec.getMessage() for rec in caplog.records if rec.name == "hfmarketdata.access"]
188 + assert lines, "no access log line"
189 + rec = json.loads(lines[-1])
190 + assert rec["request_id"] == "log-test-1" and rec["method"] == "GET" and rec["path"] == "/v1/bars/stock/AAPL"
191 + assert rec["status"] == 200 and rec["rows"] == 7 and rec["bytes"] > 0 and rec["duration_ms"] >= 0
192 + assert "principal" in rec and rec["query"].startswith("timeframe=1day")
193 +
194 +
195 +def test_security_headers_on_json(client):
196 + r = client.get("/v1/options/quarters")
197 + assert r.headers["Strict-Transport-Security"] == "max-age=31536000; includeSubDomains"
198 + assert r.headers["X-Content-Type-Options"] == "nosniff"
199 + assert r.headers["Referrer-Policy"] == "strict-origin-when-cross-origin"
200 + assert r.headers["X-Frame-Options"] == "DENY"
201 + assert "Content-Security-Policy" not in r.headers
202 +
203 +
204 +def test_swagger_and_redoc_moved(client):
205 + r = client.get("/swagger")
206 + assert r.status_code == 200 and "swagger-ui" in r.text
207 + assert "Content-Security-Policy" in r.headers and "cdn.jsdelivr.net" in r.headers["Content-Security-Policy"]
208 + assert "X-Frame-Options" not in r.headers
209 + r = client.get("/redoc")
210 + assert r.status_code == 200 and "redoc" in r.text.lower()
211 +
212 +
213 +def test_docs_is_not_swagger(client):
214 + """Without a built front-end the shell is absent (404 JSON) — but never Swagger UI."""
215 + r = client.get("/docs")
216 + assert "swagger-ui" not in r.text
217 + assert r.status_code == 404
218 +
219 +
220 +def test_openapi_etag_and_cache(client):
221 + r = client.get("/openapi.json")
222 + assert r.status_code == 200
223 + etag = r.headers["ETag"]
224 + assert etag.startswith('"') and r.headers["Cache-Control"].startswith("public")
225 + spec = r.json()
226 + assert spec["openapi"] == "3.1.0" and "/v1/bars/{asset}" in spec["paths"]
227 + r2 = client.get("/openapi.json", headers={"If-None-Match": etag})
228 + assert r2.status_code == 304 and r2.headers["ETag"] == etag
229 +
230 +
231 +def test_openapi_documents_new_errors_and_headers(client):
232 + spec = client.get("/openapi.json").json()
233 + op = spec["paths"]["/v1/bars/{asset}"]["get"]
234 + assert "X-Missing-Tickers" in op["responses"]["200"]["headers"]
235 + assert "X-Request-ID" in op["responses"]["200"]["headers"]
236 + assert "TOO_MANY_TICKERS" in op["responses"]["400"]["description"]
237 + assert "TOO_MANY_TICKERS" in spec["components"]["schemas"]["Error"]["properties"]["error"]["properties"]["code"]["enum"]
238 +
239 +
240 +def test_history_cache_control(client):
241 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01&end=2024-01-31")
242 + assert r.status_code == 200 and r.headers["Cache-Control"] == "public, max-age=86400"
243 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01") # open-ended → not cacheable
244 + assert "Cache-Control" not in r.headers
245 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day&start=2024-01-01&end=2999-01-01")
246 + assert "Cache-Control" not in r.headers
247 + r = client.get("/v1/bars/stock?tickers=AAPL,MSFT&timeframe=1day&start=2024-01-01&end=2024-01-31")
248 + assert r.headers["Cache-Control"] == "public, max-age=86400"
249 +
250 +
251 +def test_login_redirects_to_signin(client):
252 + r = client.get("/login", follow_redirects=False)
253 + assert r.status_code == 301 and r.headers["Location"] == "/signin"
254 +
255 +
256 +# ------------------------------------------------------------------------------------------------- /health
257 +
258 +
259 +def test_health_extended(client):
260 + r = client.get("/health")
261 + assert r.status_code == 200, r.text
262 + body = r.json()
263 + assert body["status"] == "ok" and body["data_root_present"] is True
264 + checks = body["checks"]
265 + assert checks["parquet"]["ok"] and checks["sqlite"]["ok"] and checks["redis"]["ok"] and checks["duckdb"]["ok"]
266 + assert checks["redis"].get("backend") == "fakeredis"
267 + assert checks["duckdb"]["rows"] > 0 and checks["duckdb"]["file"].endswith(".parquet")
268 + assert "X-RateLimit-Limit-Requests" not in r.headers # exempt from quotas
269 + assert r.headers["Cache-Control"] == "no-store"
270 +
271 +
272 +def test_health_degraded_is_503(client, monkeypatch):
273 + from core import health
274 + monkeypatch.setattr(health, "_check_sqlite", lambda: {"ok": False, "error": "boom"})
275 + r = client.get("/health")
276 + assert r.status_code == 503 and r.json()["status"] == "degraded"
277 + assert r.json()["checks"]["sqlite"]["error"] == "boom"
278 +
279 +
280 +def test_status_is_quota_exempt(client):
281 + r = client.get("/v1/status")
282 + assert r.status_code == 200
283 + # the request is counted, the rows are not
284 + assert r.headers.get("X-RateLimit-Remaining-Rows") == r.headers.get("X-RateLimit-Limit-Rows")
285 +
286 +
287 +# --------------------------------------------------------------------------------------- v1 errors unchanged
288 +
289 +
290 +def test_unknown_asset_404_asset_code(client):
291 + r = client.get("/v1/bars/bond/XYZ")
292 + assert r.status_code == 404 and r.json()["error"]["code"] == "ASSET_NOT_FOUND"
293 +
294 +
295 +def test_unknown_timeframe_400(client):
296 + r = client.get("/v1/bars/stock/AAPL?timeframe=2min")
297 + assert r.status_code == 400 and r.json()["error"]["code"] == "INVALID_PARAMETER"
298 +
299 +
300 +def test_duckdb_error_never_leaks_traceback(client, monkeypatch):
301 + import duckdb
302 + import main
303 +
304 + class _Boom:
305 + def execute(self, *a, **k):
306 + raise duckdb.ConversionException("Conversion Error: invalid timestamp field format")
307 +
308 + monkeypatch.setattr(main, "con", lambda: _Boom())
309 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day")
310 + assert r.status_code == 400
311 + assert r.json()["error"]["code"] == "INVALID_PARAMETER" and "Traceback" not in r.text
312 +
313 + class _IO:
314 + def execute(self, *a, **k):
315 + raise duckdb.IOException("IO Error: No such file")
316 +
317 + monkeypatch.setattr(main, "con", lambda: _IO())
318 + r = client.get("/v1/bars/stock/AAPL?timeframe=1day")
319 + assert r.status_code == 503 and r.json()["error"]["code"] == "SERVICE_UNAVAILABLE"
added tests/test_spa.py +109 −0
@@ -0,0 +1,109 @@
1 +"""SPA fallback (core/spa.py) on a bare FastAPI app with a fake Vite dist: known routes → 200 shell, unknown → 404 shell,
2 +real files served, hashed assets immutable + gzipped, API paths never answered by the shell."""
3 +from __future__ import annotations
4 +
5 +import pytest
6 +from fastapi import FastAPI
7 +from fastapi.testclient import TestClient
8 +from starlette.middleware.gzip import GZipMiddleware
9 +
10 +
11 +@pytest.fixture(scope="module")
12 +def spa_client(tmp_path_factory, app):
13 + from core import errors, http, spa
14 + dist = tmp_path_factory.mktemp("dist")
15 + (dist / "index.html").write_text("<!doctype html><html><body><div id=root></div><script>/*shell*/</script></body></html>" + " " * 1500)
16 + (dist / "assets").mkdir()
17 + (dist / "assets" / "index-abc123.js").write_text("console.log('hfmd');" * 200)
18 + (dist / "favicon.svg").write_text("<svg xmlns='http://www.w3.org/2000/svg'></svg>")
19 + (dist / "docs").mkdir()
20 + (dist / "docs" / "index.html").write_text("<!doctype html><html><body>prerendered docs</body></html>")
21 + (dist / "limits.html").write_text("<!doctype html><html><body>prerendered limits</body></html>")
22 + a = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
23 + errors.install(a)
24 + a.add_middleware(GZipMiddleware, minimum_size=1024)
25 +
26 + @a.get("/v1/ping")
27 + def ping():
28 + return {"ok": True}
29 +
30 + @a.get("/health")
31 + def health():
32 + return {"status": "ok"}
33 +
34 + http.install(a)
35 + assert spa.install(a, dist) is True
36 + with TestClient(a) as c:
37 + yield c
38 +
39 +
40 +def test_root_and_known_routes_serve_shell_200(spa_client):
41 + for path in ("/", "/playground", "/signin", "/dashboard", "/dashboard/keys", "/admin/users", "/integrations/mcp", "/pricing"):
42 + r = spa_client.get(path)
43 + assert r.status_code == 200, path
44 + assert "text/html" in r.headers["content-type"] and "<div id=root>" in r.text
45 + assert r.headers["Cache-Control"] == "no-cache"
46 +
47 +
48 +def test_prerendered_shells_served(spa_client):
49 + r = spa_client.get("/docs")
50 + assert r.status_code == 200 and "prerendered docs" in r.text
51 + r = spa_client.get("/docs/errors") # no prerender → shell, still a known route
52 + assert r.status_code == 200 and "<div id=root>" in r.text
53 + r = spa_client.get("/limits")
54 + assert r.status_code == 200 and "prerendered limits" in r.text
55 +
56 +
57 +def test_unknown_route_is_404_with_shell(spa_client):
58 + r = spa_client.get("/this-page-does-not-exist")
59 + assert r.status_code == 404
60 + assert "<div id=root>" in r.text and "text/html" in r.headers["content-type"]
61 + r = spa_client.get("/dashboardx")
62 + assert r.status_code == 404
63 +
64 +
65 +def test_real_files_served(spa_client):
66 + r = spa_client.get("/favicon.svg")
67 + assert r.status_code == 200 and "svg" in r.headers["content-type"]
68 +
69 +
70 +def test_assets_immutable_and_gzipped(spa_client):
71 + r = spa_client.get("/assets/index-abc123.js", headers={"Accept-Encoding": "gzip"})
72 + assert r.status_code == 200
73 + assert r.headers["Cache-Control"] == "public, max-age=31536000, immutable"
74 + assert r.headers.get("Content-Encoding") == "gzip"
75 + assert r.headers["X-Frame-Options"] == "DENY"
76 + r = spa_client.get("/assets/missing-000.js")
77 + assert r.status_code == 404
78 + assert "Cache-Control" not in r.headers or "immutable" not in r.headers["Cache-Control"]
79 +
80 +
81 +def test_html_gets_csp_not_frame_options(spa_client):
82 + r = spa_client.get("/")
83 + csp = r.headers["Content-Security-Policy"]
84 + assert "default-src 'self'" in csp and "frame-ancestors 'none'" in csp and "cdn.jsdelivr.net" not in csp
85 + assert "X-Frame-Options" not in r.headers
86 + assert r.headers["Strict-Transport-Security"].startswith("max-age=")
87 +
88 +
89 +def test_api_paths_never_get_the_shell(spa_client):
90 + r = spa_client.get("/v1/does-not-exist")
91 + assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
92 + assert r.json()["error"]["code"] == "NOT_FOUND"
93 + r = spa_client.get("/v1/ping")
94 + assert r.status_code == 200 and r.json() == {"ok": True}
95 + r = spa_client.get("/openapi.json")
96 + assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
97 + r = spa_client.get("/swagger")
98 + assert r.status_code == 404 and r.headers["content-type"].startswith("application/json")
99 +
100 +
101 +def test_login_redirect(spa_client):
102 + r = spa_client.get("/login", follow_redirects=False)
103 + assert r.status_code == 301 and r.headers["Location"] == "/signin"
104 +
105 +
106 +def test_path_traversal_blocked(spa_client):
107 + r = spa_client.get("/../../etc/passwd")
108 + assert r.status_code in (404, 200)
109 + assert "root:" not in r.text
110