core: une seule base DuckDB par processus, cache TTL/LRU, parsing strict des paramètres, middleware requête, lifespan, santé, SPA

- duck.py : `duckdb.connect()` unique au niveau module (memory_limit HFMD_DUCK_MEMORY=4GB, threads HFMD_DUCK_THREADS=4,
  temp_directory HFMD_DUCK_TEMP, object cache) + `_db.cursor()` par thread ; helpers `topn_sql`/`topn_per_group`/`run_topn`
  (CTE MATERIALIZED → un seul scan filtré, plus de re-scan intégral par la late materialization) ; `explain`/`scan_count`
- cache.py : TTLCache thread-safe avec maxsize (HFMD_CACHE_MAXSIZE=4096, HFMD_CACHE_TTL=300) — remplace les dict sans éviction
- params.py : dates ISO 8601 (YYYY-MM-DD, espace, T, Z, décalage) → datetime typé, start <= end, listes de tickers
  (dédup, max 50 → TOO_MANY_TICKERS), `bound_limit` borné par `request.state.max_rows`
- errors.py : code TOO_MANY_TICKERS ; handler `duckdb.Error` (Conversion/Binder/… → 400, IO/OOM → 503, sinon 500 propre)
- http.py : middleware ASGI pur — X-Request-ID (réutilise l'entrant), une ligne JSON par requête (logger
  hfmarketdata.access), en-têtes de sécurité (HSTS, nosniff, Referrer-Policy, X-Frame-Options / CSP pour le HTML),
  Cache-Control immutable sur /assets/*
- lifespan.py : garde secrets par défaut en production, pool anyio borné (HFMD_THREADPOOL=8), DuckDB pré-initialisé
- health.py : Parquet + SQLite + Redis (timeout 0,3 s, fakeredis toléré) + read_parquet témoin (cache 60 s) → ok/degraded
- spa.py : fallback React (routes connues → 200, inconnues → 404 avec index.html, fichiers réels servis, chemins API
  jamais avalés, /login → 301 /signin)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 18 days ago (Sep 6, 2026) parent 02b779a

9 changed files +712 −24

added hfmarketdata/api/core/cache.py +77 −0

		`@@ -0,0 +1,77 @@`
	1	+"""Small thread-safe TTL + LRU cache (no third-party dependency).
	2	+
	3	+Used for directory scans and per-root daily frames. Entries expire after `ttl` seconds; when the cache holds
	4	+more than `maxsize` entries the least recently used one is evicted, so a long-running worker never grows
	5	+without bound (the previous dict caches had no eviction at all).
	6	+
	7	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	8	+"""
	9	+from __future__ import annotations
	10	+
	11	+import threading
	12	+import time
	13	+from collections import OrderedDict
	14	+from collections.abc import Callable
	15	+from typing import Any
	16	+
	17	+_MISSING = object()
	18	+
	19	+
	20	+class TTLCache:
	21	+ def __init__(self, maxsize: int = 4096, ttl: float = 300.0) -> None:
	22	+ self.maxsize = max(1, int(maxsize))
	23	+ self.ttl = float(ttl)
	24	+ self._data: OrderedDict[str, tuple[float, Any]] = OrderedDict()
	25	+ self._lock = threading.Lock()
	26	+ self.hits = 0
	27	+ self.misses = 0
	28	+
	29	+ def get(self, key: str, default: Any = None, ttl: float \| None = None) -> Any:
	30	+ now = time.monotonic()
	31	+ with self._lock:
	32	+ hit = self._data.get(key)
	33	+ if hit is None:
	34	+ self.misses += 1
	35	+ return default
	36	+ expires, value = hit
	37	+ if ttl is not None: # caller-specific horizon (shorter or longer than the default)
	38	+ expires = expires - self.ttl + ttl
	39	+ if expires <= now:
	40	+ del self._data[key]
	41	+ self.misses += 1
	42	+ return default
	43	+ self._data.move_to_end(key)
	44	+ self.hits += 1
	45	+ return value
	46	+
	47	+ def set(self, key: str, value: Any) -> None:
	48	+ now = time.monotonic()
	49	+ with self._lock:
	50	+ self._data[key] = (now + self.ttl, value)
	51	+ self._data.move_to_end(key)
	52	+ while len(self._data) > self.maxsize:
	53	+ self._data.popitem(last=False)
	54	+
	55	+ def get_or_build(self, key: str, builder: Callable[[], Any], ttl: float \| None = None) -> Any:
	56	+ """Return the cached value or build it (outside the lock — builders may take seconds)."""
	57	+ value = self.get(key, _MISSING, ttl=ttl)
	58	+ if value is not _MISSING:
	59	+ return value
	60	+ value = builder()
	61	+ self.set(key, value)
	62	+ return value
	63	+
	64	+ def invalidate(self, prefix: str = "") -> int:
	65	+ with self._lock:
	66	+ keys = [k for k in self._data if k.startswith(prefix)]
	67	+ for k in keys:
	68	+ del self._data[k]
	69	+ return len(keys)
	70	+
	71	+ def __len__(self) -> int:
	72	+ with self._lock:
	73	+ return len(self._data)
	74	+
	75	+ def stats(self) -> dict[str, int]:
	76	+ with self._lock:
	77	+ return {"size": len(self._data), "maxsize": self.maxsize, "hits": self.hits, "misses": self.misses}

modified hfmarketdata/api/core/config.py +10 −0

		`@@ -36,6 +36,16 @@ class Settings:`
36	36	# SEC EDGAR
37	37	sec_user_agent: str = field(default_factory=lambda: _env("HFMD_SEC_USER_AGENT", "HF Market Data (Simon-Pierre Boucher, contact@spboucher.ai)"))
38	38	env: str = field(default_factory=lambda: _env("HFMD_ENV", "production"))
	39	+ # DuckDB — ONE database per process (cursors per thread), bounded memory / threads
	40	+ duck_memory: str = field(default_factory=lambda: _env("HFMD_DUCK_MEMORY", "4GB"))
	41	+ duck_threads: int = field(default_factory=lambda: int(_env("HFMD_DUCK_THREADS", "4")))
	42	+ duck_temp: Path = field(default_factory=lambda: Path(_env("HFMD_DUCK_TEMP", str(
	43	+ Path(_env("HFMD_STATE_DB", str(Path(_env("HFMD_DATA_ROOT", "/Volumes/ssd/firstratedata")) / "state" / "hfmd.db"))).parent / "duck_tmp"))))
	44	+ # anyio worker threads available to sync endpoints (each one holds a DuckDB cursor)
	45	+ threadpool: int = field(default_factory=lambda: int(_env("HFMD_THREADPOOL", "8")))
	46	+ # in-process TTL caches (directory scans, daily frames): entries / seconds
	47	+ cache_maxsize: int = field(default_factory=lambda: int(_env("HFMD_CACHE_MAXSIZE", "4096")))
	48	+ cache_ttl: int = field(default_factory=lambda: int(_env("HFMD_CACHE_TTL", "300")))
39	49
40	50	@property
41	51	def parquet(self) -> Path:

modified hfmarketdata/api/core/duck.py +114 −23

		`@@ -1,49 +1,140 @@`
1		−"""DuckDB access to the Parquet lake — one read-only connection per thread + a small TTL cache.
	1	+"""DuckDB access to the Parquet lake — ONE database per process, one cursor per thread, TTL cache.
	2	+
	3	+Why one database: `duckdb.connect()` without a path creates an independent in-memory database, each with its
	4	+own buffer pool and `memory_limit` (default = 80 % of RAM). The previous per-thread `connect()` gave every
	5	+anyio worker thread (dozens) its own database → multi-GB RSS per uvicorn worker. `_db.cursor()` shares the
	6	+buffer manager, the object cache (Parquet metadata) and the settings, while staying thread-safe.
	7	+
	8	+Settings (all `HFMD_*`, see core.config): `memory_limit` (HFMD_DUCK_MEMORY, 4GB), `threads`
	9	+(HFMD_DUCK_THREADS, 4), `temp_directory` (HFMD_DUCK_TEMP), `enable_object_cache=true`.
	10	+
	11	+`topn_per_group()` is the ONLY way to write "last/first N bars per ticker" queries: a bare
	12	+`SELECT * FROM read_parquet(...) WHERE … QUALIFY row_number() OVER (…) <= n LIMIT k` is rewritten by the
	13	+late-materialization optimizer into a second, UNFILTERED scan of every file joined back on row ids
	14	+(measured in production: 76 M rows scanned, 2.2 s CPU for 225 rows). Materializing the filtered source in a
	15	+CTE keeps a single filtered scan (0.27 s, 232 rows scanned).
2	16
3	17	Author: Simon-Pierre Boucher <contact@spboucher.ai>
4	18	"""
5	19	from __future__ import annotations
6	20
	21	+import logging
7	22	import threading
8		−import time
9	23	from collections.abc import Callable
10	24	from typing import Any
11	25
12	26	import duckdb
13	27
	28	+from .cache import TTLCache
14	29	from .config import settings
15	30
	31	+log = logging.getLogger("hfmarketdata.duck")
	32	+
16	33	_tls = threading.local()
17		−_cache: dict[str, tuple[float, Any]] = {}
18		−_cache_lock = threading.Lock()
19		−CACHE_TTL = 300
	34	+_db: duckdb.DuckDBPyConnection \| None = None
	35	+_db_lock = threading.Lock()
	36	+CACHE_TTL = settings.cache_ttl
	37	+_cache = TTLCache(maxsize=settings.cache_maxsize, ttl=CACHE_TTL)
	38	+
	39	+
	40	+def database() -> duckdb.DuckDBPyConnection:
	41	+ """The process-wide DuckDB database (created lazily, configured once)."""
	42	+ global _db
	43	+ if _db is None:
	44	+ with _db_lock:
	45	+ if _db is None:
	46	+ db = duckdb.connect()
	47	+ try:
	48	+ settings.duck_temp.mkdir(parents=True, exist_ok=True)
	49	+ db.execute(f"SET temp_directory = '{str(settings.duck_temp).replace(chr(39), chr(39) * 2)}'")
	50	+ except Exception as e: # pragma: no cover — read-only FS: DuckDB falls back to its default
	51	+ log.warning("DuckDB temp_directory %s not usable: %s", settings.duck_temp, e)
	52	+ db.execute(f"SET memory_limit = '{settings.duck_memory}'")
	53	+ db.execute(f"SET threads TO {int(settings.duck_threads)}")
	54	+ db.execute("SET enable_object_cache = true")
	55	+ _db = db
	56	+ return _db
20	57
21	58
22	59	def con() -> duckdb.DuckDBPyConnection:
23		− if not hasattr(_tls, "con"):
24		− c = duckdb.connect()
25		− c.execute("SET threads TO 4")
26		− c.execute("SET enable_object_cache=true")
	60	+ """Thread-local cursor on the shared database (DuckDB connections are not thread-safe, cursors are cheap)."""
	61	+ c = getattr(_tls, "con", None)
	62	+ if c is None:
	63	+ c = database().cursor()
27	64	_tls.con = c
28		− return _tls.con
	65	+ return c
29	66
30	67
31		−def cached(key: str, builder: Callable[[], Any], ttl: int = CACHE_TTL) -> Any:
32		− now = time.time()
33		− with _cache_lock:
34		− hit = _cache.get(key)
35		− if hit and now - hit[0] < ttl:
36		− return hit[1]
37		− value = builder()
38		− with _cache_lock:
39		− _cache[key] = (now, value)
40		− return value
	68	+def reset_for_tests() -> None: # pragma: no cover
	69	+ global _db
	70	+ with _db_lock:
	71	+ _db = None
	72	+ _tls.__dict__.clear()
	73	+ _cache.invalidate()
	74	+
	75	+
	76	+def cached(key: str, builder: Callable[[], Any], ttl: int \| None = None) -> Any:
	77	+ """TTL (+ LRU maxsize) cache for directory scans and other expensive, idempotent builders."""
	78	+ return _cache.get_or_build(key, builder, ttl=ttl)
41	79
42	80
43	81	def invalidate(prefix: str = "") -> None:
44		− with _cache_lock:
45		− for k in [k for k in _cache if k.startswith(prefix)]:
46		− del _cache[k]
	82	+ _cache.invalidate(prefix)
	83	+
	84	+
	85	+def cache_stats() -> dict[str, int]:
	86	+ return _cache.stats()
	87	+
	88	+
	89	+# ---- top-N per group without the late-materialization double scan -----------------------------------------
	90	+
	91	+def topn_sql(, source: str, partition: str, order: str, where: str = "", n: str \| int = "?", select: str = "",
	92	+ order_by: str = "") -> str:
	93	+ """SQL text: `WITH f AS MATERIALIZED (SELECT * FROM <source> [WHERE <where>])
	94	+ SELECT <select> FROM f QUALIFY row_number() OVER (PARTITION BY <partition> ORDER BY <order>) <= <n> [ORDER BY …]`.
	95	+
	96	+ `source` is a table function or a parenthesised subquery (positional `?` placeholders allowed), `where` a bare
	97	+ condition (no `WHERE` keyword), `n` a placeholder or a literal integer. Bind parameters in the order
	98	+ source → where → n."""
	99	+ w = f" WHERE {where}" if where else ""
	100	+ ob = f" ORDER BY {order_by}" if order_by else ""
	101	+ return (f"WITH f AS MATERIALIZED (SELECT * FROM {source}{w}) "
	102	+ f"SELECT {select} FROM f QUALIFY row_number() OVER (PARTITION BY {partition} ORDER BY {order}) <= {n}{ob}")
	103	+
	104	+
	105	+def topn_per_group(c: duckdb.DuckDBPyConnection, paths: list[str] \| str, where_sql: str, params: list[Any],
	106	+ partition: str, order_sql: str, n: int, , select: str = "", order_by: str = "",
	107	+ limit: int \| None = None) -> str:
	108	+ """Build the top-N-per-group query over `read_parquet(paths)` and return the SQL; the bound parameter list is
	109	+ `[paths, *params, n(, limit)]` — see `run_topn` for the one-liner that executes it.
	110	+
	111	+ `where_sql` is a bare condition (may be empty), `params` its bound values in order."""
	112	+ sql = topn_sql(source="read_parquet(?)", where=where_sql, partition=partition, order=order_sql, n="?",
	113	+ select=select, order_by=order_by)
	114	+ if limit is not None:
	115	+ sql += " LIMIT ?"
	116	+ return sql
	117	+
	118	+
	119	+def run_topn(c: duckdb.DuckDBPyConnection, paths: list[str] \| str, where_sql: str, params: list[Any],
	120	+ partition: str, order_sql: str, n: int, , select: str = "", order_by: str = "",
	121	+ limit: int \| None = None):
	122	+ """Execute `topn_per_group` and return a pandas DataFrame."""
	123	+ sql = topn_per_group(c, paths, where_sql, params, partition, order_sql, n, select=select, order_by=order_by, limit=limit)
	124	+ bound: list[Any] = [paths, *params, int(n)]
	125	+ if limit is not None:
	126	+ bound.append(int(limit))
	127	+ return c.execute(sql, bound).df()
	128	+
	129	+
	130	+def scan_count(plan: str) -> int:
	131	+ """Number of table-function scans in an EXPLAIN plan (each READ_PARQUET operator prints one `Function:` line)."""
	132	+ return plan.count("Function:")
	133	+
	134	+
	135	+def explain(c: duckdb.DuckDBPyConnection, sql: str, params: list[Any]) -> str:
	136	+ rows = c.execute("EXPLAIN " + sql, params).fetchall()
	137	+ return "\n".join(str(r[-1]) for r in rows)
47	138
48	139
49	140	PARQUET = settings.parquet

modified hfmarketdata/api/core/errors.py +34 −1

		`@@ -13,17 +13,22 @@ Author: Simon-Pierre Boucher <contact@spboucher.ai>`
13	13	"""
14	14	from __future__ import annotations
15	15
	16	+import logging
16	17	from typing import Any
17	18
	19	+import duckdb
18	20	from fastapi import FastAPI, HTTPException, Request
19	21	from fastapi.exceptions import RequestValidationError
20	22	from fastapi.responses import JSONResponse
21	23
22	24	from .config import settings
23	25
	26	+_log = logging.getLogger("hfmarketdata.errors")
	27	+
24	28	# Canonical codes (documented in /docs/errors). Add new ones here so the docs stay exhaustive.
25	29	CODES: dict[str, str] = {
26	30	"INVALID_PARAMETER": "A query or path parameter is malformed or out of range.",
	31	+ "TOO_MANY_TICKERS": "More than 50 tickers in one multi-ticker request; split the list.",
27	32	"VALIDATION_ERROR": "The request did not match the endpoint schema.",
28	33	"NOT_FOUND": "The requested resource does not exist.",
29	34	"TICKER_NOT_FOUND": "Unknown ticker for this asset type / timeframe / adjustment.",
		`@@ -131,8 +136,36 @@ def install(app: FastAPI) -> None:`
131	136	{"loc": [str(x) for x in e.get("loc", [])], "msg": e.get("msg"), "type": e.get("type")} for e in errs]})
132	137	return err.response()
133	138
	139	+ @app.exception_handler(duckdb.Error)
	140	+ async def _duckdb_error(request: Request, exc: duckdb.Error):
	141	+ return duckdb_error_response(request, exc)
	142	+
134	143	@app.exception_handler(Exception)
135		− async def _unhandled(_: Request, exc: Exception): # pragma: no cover
	144	+ async def _unhandled(request: Request, exc: Exception): # pragma: no cover
	145	+ _log.error("unhandled error on %s %s [%s]: %s", request.method, request.url.path,
	146	+ getattr(request.state, "request_id", "-"), exc, exc_info=exc)
136	147	err = ApiError(500, "INTERNAL_ERROR", "Unexpected server error. Please retry or contact "
137	148	f"{settings.contact_email}.")
138	149	return err.response()
	150	+
	151	+
	152	+# DuckDB failures never leak a traceback: bad values → 400, storage/memory trouble → 503, the rest → 500.
	153	+_DUCK_400 = (duckdb.ConversionException, duckdb.BinderException, duckdb.InvalidInputException, duckdb.ParserException,
	154	+ duckdb.OutOfRangeException, duckdb.InvalidTypeException)
	155	+_DUCK_503 = (duckdb.IOException, duckdb.OutOfMemoryException, duckdb.SerializationException)
	156	+
	157	+
	158	+def duckdb_error_response(request: Request, exc: duckdb.Error) -> JSONResponse:
	159	+ rid = getattr(request.state, "request_id", "-")
	160	+ first = str(exc).strip().splitlines()[0] if str(exc).strip() else exc.__class__.__name__
	161	+ if isinstance(exc, _DUCK_400):
	162	+ _log.info("duckdb 400 on %s [%s]: %s", request.url.path, rid, first)
	163	+ return ApiError(400, "INVALID_PARAMETER", f"Invalid parameter value: {first}",
	164	+ details={"request_id": rid}).response()
	165	+ if isinstance(exc, _DUCK_503):
	166	+ _log.error("duckdb 503 on %s [%s]: %s", request.url.path, rid, first)
	167	+ return ApiError(503, "SERVICE_UNAVAILABLE", "The data lake is temporarily unavailable. Please retry.",
	168	+ details={"request_id": rid}, headers={"Retry-After": "5"}).response()
	169	+ _log.error("duckdb 500 on %s [%s]: %s", request.url.path, rid, first, exc_info=exc)
	170	+ return ApiError(500, "INTERNAL_ERROR", f"Query failed. Please retry or contact {settings.contact_email} "
	171	+ f"with request id {rid}.", details={"request_id": rid}).response()

added hfmarketdata/api/core/health.py +94 −0

		`@@ -0,0 +1,94 @@`
	1	+"""`/health` checks: Parquet lake, SQLite state DB, Redis, a witness `read_parquet` (cached 60 s).
	2	+
	3	+`{"status": "ok" \| "degraded", "checks": {...}, "version": "..."}` — 200 when everything essential is fine,
	4	+503 otherwise. Redis is optional for serving data (the rate limiter fails open), so a Redis failure marks the
	5	+service `degraded` but still returns 503 so the monitoring notices; `HFMD_RATELIMIT=0` skips the Redis check.
	6	+
	7	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	8	+"""
	9	+from __future__ import annotations
	10	+
	11	+import os
	12	+import time
	13	+from typing import Any
	14	+
	15	+from . import db, duck
	16	+from .config import settings
	17	+
	18	+WITNESS_TTL_S = 60
	19	+REDIS_TIMEOUT_S = 0.3
	20	+
	21	+
	22	+def _check_parquet() -> dict[str, Any]:
	23	+ root = settings.parquet
	24	+ ok = root.is_dir()
	25	+ return {"ok": ok, "path": str(root)}
	26	+
	27	+
	28	+def _check_sqlite() -> dict[str, Any]:
	29	+ return {"ok": bool(db.healthcheck()), "path": str(settings.state_db)}
	30	+
	31	+
	32	+def _check_redis() -> dict[str, Any]:
	33	+ if not settings.ratelimit_enabled:
	34	+ return {"ok": True, "skipped": "rate limiting disabled"}
	35	+ url = settings.redis_url
	36	+ try:
	37	+ if url.startswith("fakeredis://"):
	38	+ import fakeredis # test / CI only
	39	+ fakeredis.FakeRedis().ping()
	40	+ return {"ok": True, "backend": "fakeredis"}
	41	+ import redis
	42	+ client = redis.from_url(url, socket_timeout=REDIS_TIMEOUT_S, socket_connect_timeout=REDIS_TIMEOUT_S)
	43	+ try:
	44	+ client.ping()
	45	+ finally:
	46	+ try:
	47	+ client.close()
	48	+ except Exception:
	49	+ pass
	50	+ return {"ok": True}
	51	+ except Exception as e:
	52	+ return {"ok": False, "error": f"{e.__class__.__name__}: {str(e)[:120]}"}
	53	+
	54	+
	55	+def _witness_file() -> str \| None:
	56	+ """One small daily file of the lake, cached (directory scans are the slow part)."""
	57	+ def find() -> str \| None:
	58	+ root = settings.parquet
	59	+ for asset in ("stock", "etf", "index", "crypto", "fx", "futures"):
	60	+ for adj_dir in sorted((root / asset / "1day").glob("*")) if (root / asset / "1day").is_dir() else []:
	61	+ try:
	62	+ with os.scandir(adj_dir) as it:
	63	+ for e in it:
	64	+ if e.name.endswith(".parquet"):
	65	+ return e.path
	66	+ except OSError:
	67	+ continue
	68	+ return None
	69	+ return duck.cached("health\|witness_file", find, ttl=WITNESS_TTL_S)
	70	+
	71	+
	72	+def _check_duckdb() -> dict[str, Any]:
	73	+ def probe() -> dict[str, Any]:
	74	+ path = _witness_file()
	75	+ if not path:
	76	+ return {"ok": False, "error": "no parquet file found under the lake"}
	77	+ t = time.perf_counter()
	78	+ try:
	79	+ n = duck.con().execute("SELECT count(*) FROM read_parquet(?)", [path]).fetchone()[0]
	80	+ except Exception as e:
	81	+ return {"ok": False, "error": f"{e.__class__.__name__}: {str(e)[:120]}", "file": os.path.basename(path)}
	82	+ return {"ok": True, "file": os.path.basename(path), "rows": int(n), "ms": round((time.perf_counter() - t) * 1000, 1),
	83	+ "memory_limit": settings.duck_memory, "threads": settings.duck_threads}
	84	+ return duck.cached("health\|duckdb", probe, ttl=WITNESS_TTL_S)
	85	+
	86	+
	87	+def report(version: str) -> tuple[dict[str, Any], int]:
	88	+ checks = {"parquet": _check_parquet(), "sqlite": _check_sqlite(), "redis": _check_redis()}
	89	+ checks["duckdb"] = _check_duckdb() if checks["parquet"]["ok"] else {"ok": False, "error": "lake missing"}
	90	+ ok = all(c.get("ok") for c in checks.values())
	91	+ body = {"status": "ok" if ok else "degraded", "service": "hfmarketdata-api", "version": version,
	92	+ "data_root_present": checks["parquet"]["ok"], # legacy field
	93	+ "checks": checks, "cache": duck.cache_stats()}
	94	+ return body, (200 if ok else 503)

added hfmarketdata/api/core/http.py +162 −0

		`@@ -0,0 +1,162 @@`
	1	+"""Pure ASGI request-context middleware: `X-Request-ID`, one JSON access-log line per request, security headers,
	2	+immutable caching of the hashed front-end chunks.
	3	+
	4	+No `BaseHTTPMiddleware` (it buffers bodies and breaks streaming): the middleware only touches the
	5	+`http.response.start` message and counts body bytes as they pass.
	6	+
	7	+Access log (`hfmarketdata.access`, one JSON object per line):
	8	+ {"ts": "...Z", "request_id": "…", "method": "GET", "path": "/v1/bars/stock", "status": 200,
	9	+ "duration_ms": 12.3, "rows": 225, "bytes": 18234, "principal": "key:12" \| null, "ip": "…"}
	10	+uvicorn's own access log is redundant → run with `--no-access-log`.
	11	+
	12	+Security headers on every response: HSTS, `X-Content-Type-Options: nosniff`, `Referrer-Policy`, and
	13	+`X-Frame-Options: DENY` for non-HTML responses; HTML pages get a Content-Security-Policy instead (the SPA is
	14	+self-hosted — lightweight-charts is bundled, no CDN; Swagger/ReDoc need cdn.jsdelivr.net).
	15	+
	16	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	17	+"""
	18	+from __future__ import annotations
	19	+
	20	+import json
	21	+import logging
	22	+import re
	23	+import sys
	24	+import time
	25	+import uuid
	26	+from datetime import datetime, timezone
	27	+
	28	+from fastapi import FastAPI
	29	+from starlette.datastructures import Headers, MutableHeaders
	30	+
	31	+access_log = logging.getLogger("hfmarketdata.access")
	32	+
	33	+REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,64}$")
	34	+ASSETS_PREFIX = "/assets/"
	35	+ASSETS_CACHE = "public, max-age=31536000, immutable"
	36	+
	37	+SECURITY_HEADERS = {
	38	+ "strict-transport-security": "max-age=31536000; includeSubDomains",
	39	+ "x-content-type-options": "nosniff",
	40	+ "referrer-policy": "strict-origin-when-cross-origin",
	41	+}
	42	+# React platform (index.html has one inline theme bootstrap script → 'unsafe-inline' for scripts/styles)
	43	+CSP_SPA = ("default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; "
	44	+ "img-src 'self' data: blob: https:; font-src 'self' data:; connect-src 'self' https: wss:; "
	45	+ "worker-src 'self' blob:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'; object-src 'none'")
	46	+# Swagger UI / ReDoc are loaded from jsdelivr by FastAPI's default HTML
	47	+CSP_DOCS_UI = ("default-src 'self'; script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
	48	+ "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
	49	+ "img-src 'self' data: https:; font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com; "
	50	+ "connect-src 'self' https:; worker-src 'self' blob:; child-src blob:; frame-ancestors 'none'; object-src 'none'")
	51	+DOCS_UI_PATHS = ("/swagger", "/redoc")
	52	+
	53	+
	54	+def new_request_id() -> str:
	55	+ return uuid.uuid4().hex[:16]
	56	+
	57	+
	58	+def _client_ip(scope, headers: Headers) -> str \| None:
	59	+ fwd = headers.get("x-forwarded-for")
	60	+ if fwd:
	61	+ return fwd.split(",")[0].strip()
	62	+ client = scope.get("client")
	63	+ return client[0] if client else None
	64	+
	65	+
	66	+class RequestContextMiddleware:
	67	+ def __init__(self, app) -> None:
	68	+ self.app = app
	69	+
	70	+ async def __call__(self, scope, receive, send):
	71	+ if scope["type"] != "http":
	72	+ return await self.app(scope, receive, send)
	73	+ headers_in = Headers(scope=scope)
	74	+ incoming = headers_in.get("x-request-id")
	75	+ request_id = incoming if incoming and REQUEST_ID_RE.match(incoming) else new_request_id()
	76	+ state: dict = scope.setdefault("state", {})
	77	+ state["request_id"] = request_id
	78	+ path: str = scope.get("path", "")
	79	+ method: str = scope.get("method", "-")
	80	+ started = time.perf_counter()
	81	+ info = {"status": 0, "rows": None, "bytes": 0}
	82	+
	83	+ async def send_wrapper(message):
	84	+ if message["type"] == "http.response.start":
	85	+ info["status"] = int(message["status"])
	86	+ headers = MutableHeaders(scope=message)
	87	+ headers["X-Request-ID"] = request_id
	88	+ rows = headers.get("x-row-count")
	89	+ if rows is not None:
	90	+ try:
	91	+ info["rows"] = int(rows)
	92	+ except ValueError:
	93	+ pass
	94	+ self._decorate(path, headers, info["status"])
	95	+ elif message["type"] == "http.response.body":
	96	+ body = message.get("body")
	97	+ if body:
	98	+ info["bytes"] += len(body)
	99	+ await send(message)
	100	+
	101	+ try:
	102	+ await self.app(scope, receive, send_wrapper)
	103	+ except Exception:
	104	+ if not info["status"]:
	105	+ info["status"] = 500
	106	+ self._log(scope, headers_in, method, path, request_id, started, info)
	107	+ raise
	108	+ self._log(scope, headers_in, method, path, request_id, started, info)
	109	+
	110	+ @staticmethod
	111	+ def _decorate(path: str, headers: MutableHeaders, status: int) -> None:
	112	+ for k, v in SECURITY_HEADERS.items():
	113	+ if k not in headers:
	114	+ headers[k] = v
	115	+ ctype = (headers.get("content-type") or "").split(";")[0].strip().lower()
	116	+ if ctype == "text/html":
	117	+ if "content-security-policy" not in headers:
	118	+ headers["content-security-policy"] = CSP_DOCS_UI if path.startswith(DOCS_UI_PATHS) else CSP_SPA
	119	+ elif "x-frame-options" not in headers:
	120	+ headers["x-frame-options"] = "DENY"
	121	+ if status == 200 and path.startswith(ASSETS_PREFIX) and "cache-control" not in headers:
	122	+ headers["cache-control"] = ASSETS_CACHE
	123	+
	124	+ @staticmethod
	125	+ def _log(scope, headers_in: Headers, method: str, path: str, request_id: str, started: float, info: dict) -> None:
	126	+ if not access_log.isEnabledFor(logging.INFO):
	127	+ return
	128	+ state = scope.get("state") or {}
	129	+ rec = {
	130	+ "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
	131	+ "request_id": request_id,
	132	+ "method": method,
	133	+ "path": path,
	134	+ "status": info["status"],
	135	+ "duration_ms": round((time.perf_counter() - started) * 1000, 2),
	136	+ "rows": info["rows"],
	137	+ "bytes": info["bytes"],
	138	+ "principal": state.get("principal"),
	139	+ "ip": _client_ip(scope, headers_in),
	140	+ }
	141	+ query = scope.get("query_string") or b""
	142	+ if query:
	143	+ rec["query"] = query.decode("latin-1")[:512]
	144	+ access_log.info(json.dumps(rec, separators=(",", ":"), ensure_ascii=False))
	145	+
	146	+
	147	+def ensure_access_logger() -> None:
	148	+ """Plain stdout handler for the access logger (one JSON line per request), unless the deployment configured one."""
	149	+ if not access_log.handlers:
	150	+ h = logging.StreamHandler(sys.stdout)
	151	+ h.setFormatter(logging.Formatter("%(message)s"))
	152	+ access_log.addHandler(h)
	153	+ access_log.propagate = False
	154	+ if access_log.level == logging.NOTSET:
	155	+ access_log.setLevel(logging.INFO)
	156	+
	157	+
	158	+def install(app: FastAPI) -> None:
	159	+ """Mount as the OUTERMOST middleware (after every other add_middleware call) so the whole request is timed and
	160	+ every response — including 429/500 produced by inner layers — carries `X-Request-ID` and the security headers."""
	161	+ ensure_access_logger()
	162	+ app.add_middleware(RequestContextMiddleware)

added hfmarketdata/api/core/lifespan.py +50 −0

		`@@ -0,0 +1,50 @@`
	1	+"""Application lifespan: production guards, bounded thread pool, DuckDB warm-up.
	2	+
	3	+* Refuses to start in production with the default `HFMD_SECRET_KEY` / `HFMD_KEY_SALT` (sessions and API-key
	4	+ hashes would be forgeable).
	5	+* Caps the anyio worker pool used by sync endpoints (`HFMD_THREADPOOL`, default 8; Starlette's default is 40) —
	6	+ each worker thread holds a DuckDB cursor, and DuckDB already parallelises internally (`HFMD_DUCK_THREADS`).
	7	+* Creates the shared DuckDB database up-front so the first request does not pay for it.
	8	+
	9	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	10	+"""
	11	+from __future__ import annotations
	12	+
	13	+import contextlib
	14	+import logging
	15	+from collections.abc import AsyncIterator
	16	+
	17	+import anyio.to_thread
	18	+from fastapi import FastAPI
	19	+
	20	+from . import duck
	21	+from .config import settings
	22	+from .http import ensure_access_logger
	23	+
	24	+log = logging.getLogger("hfmarketdata")
	25	+
	26	+DEFAULT_SECRET_KEY = "dev-only-change-me"
	27	+DEFAULT_KEY_SALT = "hfmd-key-salt-dev"
	28	+
	29	+
	30	+def guard_production_secrets() -> None:
	31	+ if settings.env != "production":
	32	+ return
	33	+ bad = [name for name, value, default in (("HFMD_SECRET_KEY", settings.secret_key, DEFAULT_SECRET_KEY),
	34	+ ("HFMD_KEY_SALT", settings.key_hash_salt, DEFAULT_KEY_SALT))
	35	+ if not value or value == default]
	36	+ if bad:
	37	+ raise RuntimeError(f"Refusing to start in production with default secrets: set {', '.join(bad)} "
	38	+ "(or HFMD_ENV=development for a local run).")
	39	+
	40	+
	41	+@contextlib.asynccontextmanager
	42	+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
	43	+ guard_production_secrets()
	44	+ ensure_access_logger()
	45	+ limiter = anyio.to_thread.current_default_thread_limiter()
	46	+ limiter.total_tokens = max(1, int(settings.threadpool))
	47	+ duck.database()
	48	+ log.info("hfmarketdata ready: env=%s threadpool=%d duckdb(memory=%s, threads=%d) parquet=%s",
	49	+ settings.env, limiter.total_tokens, settings.duck_memory, settings.duck_threads, settings.parquet)
	50	+ yield

added hfmarketdata/api/core/params.py +93 −0

		`@@ -0,0 +1,93 @@`
	1	+"""Strict parsing of the v1 query parameters (dates, ticker lists, limits) → 400 INVALID_PARAMETER.
	2	+
	3	+Before this module, `start`/`end`/`at`/`trade_date`/`expiry` were bound as raw strings and DuckDB's
	4	+`ConversionException` surfaced as a 500 INTERNAL_ERROR. Every parser here raises `ApiError(400, …)` with the
	5	+parameter name in `details`, and returns a typed value that is bound as such.
	6	+
	7	+Accepted date/time forms: `YYYY-MM-DD`, `YYYY-MM-DD HH:MM[:SS[.ffffff]]`, the same with `T`, an optional `Z`
	8	+or `±HH:MM` offset. Lake timestamps are naive (US/Eastern for intraday): naive inputs are compared as-is;
	9	+tz-aware inputs are converted to UTC and the offset dropped — exactly what DuckDB's VARCHAR → TIMESTAMP cast
	10	+did before, so no behaviour changes for valid inputs.
	11	+
	12	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	13	+"""
	14	+from __future__ import annotations
	15	+
	16	+import re
	17	+from datetime import date, datetime, timezone
	18	+from typing import Any
	19	+
	20	+from .errors import ApiError
	21	+
	22	+_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
	23	+MAX_TICKERS = 50
	24	+
	25	+
	26	+def parse_datetime(value: str \| None, name: str) -> datetime \| None:
	27	+ """ISO 8601 date or datetime → naive datetime (UTC-normalised when an offset is given); None when empty."""
	28	+ if value is None:
	29	+ return None
	30	+ v = value.strip()
	31	+ if not v:
	32	+ return None
	33	+ try:
	34	+ if _DATE_RE.match(v):
	35	+ return datetime.fromisoformat(v) # midnight, same as DuckDB's cast of a bare date
	36	+ dt = datetime.fromisoformat(v.replace("Z", "+00:00").replace("z", "+00:00"))
	37	+ except ValueError:
	38	+ raise ApiError(400, "INVALID_PARAMETER",
	39	+ f"{name} must be an ISO 8601 date or datetime (e.g. 2024-06-03 or 2024-06-03 10:35:00), got '{value}'",
	40	+ details={name: value})
	41	+ if dt.tzinfo is not None:
	42	+ dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
	43	+ return dt
	44	+
	45	+
	46	+def parse_date(value: str \| None, name: str) -> date \| None:
	47	+ """ISO 8601 date (a datetime is accepted and truncated) → `date`; None when empty."""
	48	+ if value is None:
	49	+ return None
	50	+ v = value.strip()
	51	+ if not v:
	52	+ return None
	53	+ try:
	54	+ return date.fromisoformat(v) if _DATE_RE.match(v) else datetime.fromisoformat(v.replace("Z", "+00:00")).date()
	55	+ except ValueError:
	56	+ raise ApiError(400, "INVALID_PARAMETER", f"{name} must be an ISO date (YYYY-MM-DD), got '{value}'",
	57	+ details={name: value})
	58	+
	59	+
	60	+def parse_range(start: str \| None, end: str \| None) -> tuple[datetime \| None, datetime \| None]:
	61	+ """`start`/`end` bounds (both optional); `start <= end` is enforced."""
	62	+ lo, hi = parse_datetime(start, "start"), parse_datetime(end, "end")
	63	+ if lo is not None and hi is not None and lo > hi:
	64	+ raise ApiError(400, "INVALID_PARAMETER", f"start ({start}) must be on or before end ({end})",
	65	+ details={"start": start, "end": end})
	66	+ return lo, hi
	67	+
	68	+
	69	+def split_tickers(raw: str, max_n: int = MAX_TICKERS) -> list[str]:
	70	+ """Comma-separated tickers → upper-cased, de-duplicated list (order kept). 400 when empty or above `max_n`."""
	71	+ seen: dict[str, None] = {}
	72	+ for t in raw.split(","):
	73	+ t = t.strip().upper()
	74	+ if t:
	75	+ seen.setdefault(t, None)
	76	+ names = list(seen)
	77	+ if not names:
	78	+ raise ApiError(400, "INVALID_PARAMETER", "No tickers given", details={"tickers": raw})
	79	+ if len(names) > max_n:
	80	+ raise ApiError(400, "TOO_MANY_TICKERS", f"{len(names)} tickers requested; the maximum is {max_n} per call.",
	81	+ details={"max": max_n, "given": len(names)})
	82	+ return names
	83	+
	84	+
	85	+def bound_limit(limit: int, hard_max: int, request: Any = None) -> int:
	86	+ """v1 semantics: `limit` is silently clamped to the endpoint ceiling and to the tier's rows-per-request cap
	87	+ (`request.state.max_rows`, set by the rate-limit middleware) — never an error."""
	88	+ lim = max(1, int(limit))
	89	+ cap = hard_max
	90	+ tier_cap = getattr(getattr(request, "state", None), "max_rows", None) if request is not None else None
	91	+ if tier_cap:
	92	+ cap = min(cap, int(tier_cap))
	93	+ return min(lim, cap)

added hfmarketdata/api/core/spa.py +78 −0

		`@@ -0,0 +1,78 @@`
	1	+"""React platform: hashed assets, prerendered shells and the SPA fallback (`install(app, dist)`).
	2	+
	3	+* `/assets/*` — Vite's content-hashed chunks, served by StaticFiles; the request-context middleware adds
	4	+ `Cache-Control: public, max-age=31536000, immutable` and GZip compresses them.
	5	+* Real files of the dist (favicon.svg, robots.txt, `<route>.html`, `<route>/index.html`) are always served.
	6	+* Routes known to the React router get `index.html` with 200; anything else gets `index.html` with 404
	7	+ (the shell renders its own "not found" page, crawlers see the right status).
	8	+* API paths are never answered by the shell (JSON 404 instead).
	9	+* `/login` → 301 `/signin` (the SPA only knows `/signin`).
	10	+
	11	+Mount LAST: the catch-all must not shadow any API route.
	12	+
	13	+Author: Simon-Pierre Boucher <contact@spboucher.ai>
	14	+"""
	15	+from __future__ import annotations
	16	+
	17	+from pathlib import Path
	18	+
	19	+from fastapi import FastAPI
	20	+from fastapi.responses import FileResponse, RedirectResponse
	21	+from fastapi.staticfiles import StaticFiles
	22	+
	23	+from .errors import ApiError
	24	+
	25	+# Paths owned by the API — never answered by the SPA shell.
	26	+API_PREFIXES = ("v1/", "health", "openapi.json", "swagger", "redoc", "assets/")
	27	+# Routes known to the React router (web/src/App.jsx). Anything else gets the shell with a 404 status.
	28	+SPA_ROUTES = frozenset({"", "docs", "playground", "integrations", "limits", "pricing", "status", "signin", "signup",
	29	+ "verify", "reset", "invite", "accept-invite", "reset-password", "dashboard", "admin"})
	30	+SPA_PREFIXES = ("docs/", "integrations/", "dashboard/", "admin/")
	31	+SHELL_CACHE = "no-cache"
	32	+REDIRECTS = {"login": "/signin"}
	33	+
	34	+
	35	+def is_spa_route(path: str) -> bool:
	36	+ p = path.strip("/")
	37	+ return p in SPA_ROUTES or p.startswith(SPA_PREFIXES)
	38	+
	39	+
	40	+def is_api_path(path: str) -> bool:
	41	+ p = path.lstrip("/")
	42	+ return p.startswith(API_PREFIXES) or p.rstrip("/") in ("health", "swagger", "redoc", "openapi.json")
	43	+
	44	+
	45	+def _redirect(target: str):
	46	+ def handler():
	47	+ return RedirectResponse(target, status_code=301)
	48	+ return handler
	49	+
	50	+
	51	+def install(app: FastAPI, dist: Path) -> bool:
	52	+ """Mount the platform from `dist`; returns False (nothing mounted except the redirects) when `dist` is missing."""
	53	+ for src, target in REDIRECTS.items():
	54	+ app.add_api_route(f"/{src}", _redirect(target), methods=["GET"], include_in_schema=False, name=f"redirect_{src}")
	55	+ if not dist.is_dir():
	56	+ return False
	57	+ assets = dist / "assets"
	58	+ if assets.is_dir():
	59	+ app.mount("/assets", StaticFiles(directory=str(assets)), name="assets")
	60	+ index = dist / "index.html"
	61	+ root = str(dist.resolve())
	62	+
	63	+ def spa(path: str):
	64	+ if is_api_path(path):
	65	+ raise ApiError(404, "NOT_FOUND", "Not found")
	66	+ candidate = (dist / path).resolve()
	67	+ if path and str(candidate).startswith(root):
	68	+ # exact file, prerendered shell (<route>.html or <route>/index.html), else the SPA entry
	69	+ for c in (candidate, candidate.with_suffix(".html") if candidate.suffix == "" else None, candidate / "index.html"):
	70	+ if c is not None and c.is_file():
	71	+ if c.suffix == ".html":
	72	+ return FileResponse(c, headers={"Cache-Control": SHELL_CACHE})
	73	+ return FileResponse(c)
	74	+ status_code = 200 if is_spa_route(path) else 404
	75	+ return FileResponse(index, status_code=status_code, headers={"Cache-Control": SHELL_CACHE})
	76	+
	77	+ app.add_api_route("/{path:path}", spa, methods=["GET"], include_in_schema=False, name="spa")
	78	+ return True
	79