"""Site principal: the /charts page has no quota (ratelimit/middleware.py `is_site_request`). Three cumulative criteria (`X-HFMD-Client`, `Sec-Fetch-Site`, same-host `Origin`/`Referer`) on data GETs → principal `site`: no X-RateLimit-* headers, no 429 for quota reasons, max_rows 200 000, usage under `site` (never under the visitor's IP), and a per-IP burst guard that is a DoS protection, not a product limit. """ from __future__ import annotations from dataclasses import replace import pytest from fastapi import FastAPI, Request # module level: `from __future__ import annotations` + a fixture-local import # would leave `Request` unresolvable in the echo handler (→ query param, 422) SITE = {"X-HFMD-Client": "charts", "Sec-Fetch-Site": "same-origin", "Origin": "https://www.hfmarketdata.io"} RL_HDRS = ("X-RateLimit-Limit-Requests", "X-RateLimit-Remaining-Requests", "X-RateLimit-Limit-Rows", "X-RateLimit-Remaining-Rows", "X-RateLimit-Reset") def ip(n: int) -> dict: return {"X-Forwarded-For": f"172.16.0.1, 10.0.0.{n}"} def site(n: int, **over) -> dict: return {**ip(n), **SITE, **over} def no_rl_headers(r) -> bool: return not any(h in r.headers for h in RL_HDRS) # ---------------------------------------------------------------------------------------------------- # Full app (synthetic lake) # ---------------------------------------------------------------------------------------------------- def test_site_requests_are_exempt_and_leave_the_ip_counters_untouched(client): for _ in range(31): # keyless would 429 after 30 r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=5", headers=site(101)) assert r.status_code == 200, r.text assert no_rl_headers(r) assert r.headers["X-Row-Count"] == "5" # the same IP, as a plain keyless client, still has its full hourly budget r = client.get("/v1/status", headers=ip(101)) assert r.status_code == 200 and r.headers["X-RateLimit-Remaining-Requests"] == "29" assert r.headers["X-RateLimit-Remaining-Rows"] == "100000" @pytest.mark.parametrize("path", ["/v1/stock/tickers?limit=3", "/v1/futures/roots", "/v1/options/quarters", "/v1/bars/stock/AAPL?timeframe=1min&limit=10"]) def test_every_charts_data_path_is_covered(client, path): r = client.get(path, headers=site(102)) assert r.status_code == 200, r.text assert no_rl_headers(r) def test_web_client_label_and_referer_fallback_are_accepted(client): r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=2", headers=site(103, **{"X-HFMD-Client": "web"})) assert r.status_code == 200 and no_rl_headers(r) hdrs = {**ip(103), "X-HFMD-Client": "charts", "Sec-Fetch-Site": "same-site", "Referer": "https://hfmarketdata.io/charts?s=AAPL"} r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=2", headers=hdrs) assert r.status_code == 200 and no_rl_headers(r) @pytest.mark.parametrize("missing", [ {"Sec-Fetch-Site": None}, # no Sec-Fetch-Site → keyless {"Sec-Fetch-Site": "cross-site"}, {"Origin": "https://evil.example"}, # wrong host {"Origin": None}, # no Origin, no Referer {"X-HFMD-Client": None}, {"X-HFMD-Client": "curl"}, ]) def test_any_missing_criterion_falls_back_to_keyless(client, missing): hdrs = {**ip(104), **SITE} for k, v in missing.items(): if v is None: hdrs.pop(k) else: hdrs[k] = v r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=2", headers=hdrs) assert r.status_code == 200 assert r.headers["X-RateLimit-Limit-Requests"] == "30", dict(r.headers) def test_site_headers_on_a_non_data_path_or_a_non_get_are_ignored(client): r = client.get("/v1/status", headers=site(105)) assert r.headers["X-RateLimit-Limit-Requests"] == "30" # /v1/status is not a charts data path r = client.get("/v1/limits", headers=site(105)) assert r.json()["data"]["principal"]["kind"] == "keyless" def test_bearer_key_wins_over_the_site_criteria(client, make_user): _, key, _, _ = make_user() r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=2", headers=site(106, Authorization=f"Bearer {key}")) assert r.status_code == 200 and r.headers["X-RateLimit-Limit-Requests"] == "120" def test_usage_is_recorded_under_site_not_under_the_ip(client): from ratelimit import middleware, usage client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=7", headers=site(107)) live = usage._live_minutes() by_principal: dict[str, dict] = {} for (principal, _minute), fields in live.items(): acc = by_principal.setdefault(principal, {}) for k, v in fields.items(): acc[k] = acc.get(k, 0) + v assert by_principal["site"]["requests"] >= 1 and by_principal["site"]["rows"] >= 7 assert f"ip:{middleware.hash_ip('10.0.0.107')}" not in by_principal def test_burst_guard_is_a_dedicated_429_without_ratelimit_headers(client, monkeypatch): from ratelimit import tiers monkeypatch.setattr(tiers, "SITE_BURST_PER_MINUTE_PER_IP", 3) for _ in range(3): assert client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=1", headers=site(108)).status_code == 200 r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=1", headers=site(108)) assert r.status_code == 429 body = r.json()["error"] assert body["code"] == "RATE_LIMIT_EXCEEDED" and body["details"]["guard"] == "site_burst" assert "not a quota" in body["message"] and "free account" not in body["message"] assert int(r.headers["Retry-After"]) >= 1 and no_rl_headers(r) # another IP is unaffected, and the guarded IP keeps its keyless budget intact assert client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=1", headers=site(109)).status_code == 200 assert client.get("/v1/status", headers=ip(108)).headers["X-RateLimit-Remaining-Requests"] == "29" def test_burst_guard_fails_open_when_redis_is_down(client, monkeypatch): from ratelimit import redis_limiter as rl from ratelimit import tiers monkeypatch.setattr(tiers, "SITE_BURST_PER_MINUTE_PER_IP", 1) monkeypatch.setattr(rl, "throttle", lambda *a, **k: None) for _ in range(3): r = client.get("/v1/bars/stock/AAPL?timeframe=1day&limit=1", headers=site(110)) assert r.status_code == 200 and no_rl_headers(r) # ---------------------------------------------------------------------------------------------------- # Minimal app: what the handler sees (max_rows, principal), localhost rule # ---------------------------------------------------------------------------------------------------- @pytest.fixture def mini(app): """Tiny FastAPI app with only the rate-limit middleware, echoing request.state on a charts data path.""" from fastapi.testclient import TestClient from core.responses import json_response from ratelimit import middleware m = FastAPI() def echo(request: Request, asset: str, ticker: str): st = request.state return json_response({"principal": st.principal, "kind": st.principal_kind, "max_rows": st.max_rows, "ratelimit": st.ratelimit}) m.add_api_route("/v1/bars/{asset}/{ticker}", echo, methods=["GET"]) middleware.install(m) with TestClient(m) as c: yield c def test_site_state_max_rows_and_principal(mini): r = mini.get("/v1/bars/stock/AAPL", headers=site(111)) assert r.status_code == 200 body = r.json()["data"] assert body == {"principal": "site", "kind": "site", "max_rows": 200_000, "ratelimit": {"principal": "site", "kind": "site", "tier": "site", "exempt": True, "max_rows_per_request": 200_000}} r = mini.get("/v1/bars/stock/AAPL", headers=ip(111)).json()["data"] assert r["kind"] == "keyless" and r["max_rows"] == 5000 def test_localhost_origin_only_outside_production(mini, monkeypatch): from ratelimit import middleware dev = site(112, Origin="http://localhost:5173") assert mini.get("/v1/bars/stock/AAPL", headers=dev).json()["data"]["kind"] == "site" # HFMD_ENV=test monkeypatch.setattr(middleware, "settings", replace(middleware.settings, env="production")) assert mini.get("/v1/bars/stock/AAPL", headers=dev).json()["data"]["kind"] == "keyless" assert mini.get("/v1/bars/stock/AAPL", headers=site(112)).json()["data"]["kind"] == "site" # public host still fine def test_is_site_request_unit(): from starlette.datastructures import Headers from ratelimit.middleware import is_site_data_path, is_site_request assert is_site_data_path("/v1/bars/stock/AAPL") and is_site_data_path("/v1/fx/tickers") and is_site_data_path("/v1/futures/ES/contracts") assert not is_site_data_path("/v1/status") and not is_site_data_path("/v1/fundamentals/screen") and not is_site_data_path("/v1/stock/tickers/x") scope = {"method": "GET", "path": "/v1/bars/stock/AAPL", "query_string": b""} good = Headers({"x-hfmd-client": "charts", "sec-fetch-site": "same-origin", "origin": "https://www.hfmarketdata.io"}) assert is_site_request(scope, good) assert not is_site_request({**scope, "method": "POST"}, good) assert not is_site_request({**scope, "query_string": b"api_key=hfmd_live_x"}, good) assert not is_site_request(scope, Headers({"x-hfmd-client": "charts", "sec-fetch-site": "same-origin", "origin": "https://www.hfmarketdata.io.evil.example"}))