spb/hfmarketdata

1"""Shared pytest fixtures.23The tests never touch the real 350 GB lake: `tests/fixtures/make_fixtures.py` builds a tiny synthetic4Parquet lake (same layout as frd_downloader.py) in a temp directory, and the API is imported with5HFMD_DATA_ROOT pointing at it. Redis is replaced by fakeredis; the SQLite state DB lives in the temp dir.6"""7from __future__ import annotations89import importlib10import os11import sys12from pathlib import Path1314import pytest1516ROOT = Path(__file__).resolve().parents[1]17API_DIR = ROOT / "hfmarketdata" / "api"18if str(API_DIR) not in sys.path:19    sys.path.insert(0, str(API_DIR))202122@pytest.fixture(scope="session")23def lake(tmp_path_factory) -> Path:24    from tests.fixtures.make_fixtures import build_lake25    root = tmp_path_factory.mktemp("lake")26    build_lake(root)27    return root282930@pytest.fixture(scope="session")31def app(lake):32    os.environ["HFMD_DATA_ROOT"] = str(lake)33    os.environ["HFMD_STATE_DB"] = str(lake / "state" / "hfmd.db")34    os.environ["HFMD_WEB_DIST"] = str(lake / "no-web")35    os.environ["HFMD_ENV"] = "test"36    os.environ["HFMD_REDIS_URL"] = "fakeredis://"37    os.environ["HFMD_SECRET_KEY"] = "test-secret"38    os.environ["HFMD_KEY_SALT"] = "test-salt"39    for m in [m for m in list(sys.modules) if m.split(".")[0] in ("core", "main", "futures", "accounts", "ratelimit", "fundamentals", "bulk", "stream", "openapi")]:40        del sys.modules[m]41    main = importlib.import_module("main")42    from tests.fixtures.v2_test_routes import install as _install_probes43    _install_probes(main.app)   # /v1/_test/* probes for the rate-limit contract (test app only)44    return main.app454647@pytest.fixture(scope="session")48def client(app):49    from fastapi.testclient import TestClient50    with TestClient(app) as c:51        yield c525354@pytest.fixture(autouse=True)55def _fresh_counters():56    """Every test starts with empty rate-limit / usage counters and an empty key cache (fakeredis only)."""57    yield58    if "ratelimit.redis_limiter" in sys.modules:59        sys.modules["ratelimit.redis_limiter"].reset_for_tests()60    if "ratelimit.middleware" in sys.modules:61        sys.modules["ratelimit.middleware"].invalidate_key_cache()626364@pytest.fixture65def make_user(app):66    """Create an active, verified user with one active API key. Returns (user_id, raw_key, email, password)."""67    from accounts import service68    from core.db import session69    counter = {"n": 0}7071    def _make(email: str | None = None, *, tier: str = "free", role: str = "user", password: str = "correct-horse-battery",72              with_key: bool = True):73        counter["n"] += 174        import uuid75        email = email or f"user{counter['n']}-{os.getpid()}-{uuid.uuid4().hex[:10]}@example.com"   # id(counter) collided76        with session() as s:77            u = service.create_user(s, email, "Test User", password=password, tier=tier, role=role, status="active", actor="test")78            u.email_verified_at = service.now()79            raw = None80            if with_key:81                raw, _ = service.create_key(s, u, "default", actor="test", notify=False)82            uid = u.id83        return uid, raw, email, password84    return _make858687@pytest.fixture88def web(app):89    """Fresh TestClient (own cookie jar) for browser-session flows. Not shared with `client`."""90    from fastapi.testclient import TestClient91    with TestClient(app) as c:92        yield c939495@pytest.fixture96def signin(app):97    """signin(client, email, password) → logs a TestClient in through POST /v1/auth/login (cookie session)."""98    def _signin(c, email: str, password: str):99        r = c.post("/v1/auth/login", json={"email": email, "password": password}, headers={"Content-Type": "application/json"})100        assert r.status_code == 200, r.text101        return r.json()["data"]102    return _signin103104105@pytest.fixture106def admin_web(app, make_user, signin):107    """TestClient signed in (cookie) as a fresh admin. Returns (client, user_id, email, password)."""108    from fastapi.testclient import TestClient109    uid, _, email, pw = make_user(role="admin", with_key=False)110    with TestClient(app) as c:111        signin(c, email, pw)112        yield c, uid, email, pw113114115@pytest.fixture116def outbox(app, monkeypatch):117    """Capture every e-mail queued by the accounts service (provider considered configured, nothing sent)."""118    from dataclasses import replace119120    from accounts import mailer121    sent: list = []122    monkeypatch.setattr(mailer, "settings", replace(mailer.settings, resend_api_key="re_test_capture"))123    monkeypatch.setattr(mailer, "dispatch", lambda msg: sent.append(msg))124    return sent125126127@pytest.fixture128def client_hu(app, make_user):129    """Client authenticated with a high_usage API key (600 req/min, 200 000 rows/request) — for data-heavy tests."""130    from fastapi.testclient import TestClient131    _, raw, _, _ = make_user(tier="high_usage")132    with TestClient(app, headers={"Authorization": f"Bearer {raw}"}) as c:133        yield c134135# ----------------------------------------------------------------------------------------- fundamentals136EDGAR_FIXTURES = ROOT / "tests" / "fixtures" / "edgar"137PROTO_CIKS = {"AAPL": 320193, "MSFT": 789019, "SHAK": 1620533}138139140def load_gz(name: str):141    import gzip142    import json143    with gzip.open(EDGAR_FIXTURES / name, "rt", encoding="utf-8") as fh:144        return json.load(fh)145146147@pytest.fixture(scope="session")148def edgar_mock(app):149    """respx router serving the recorded EDGAR fixtures — the tests never reach sec.gov."""150    import respx151    from fundamentals import edgar_client as ec152    router = respx.mock(assert_all_called=False, assert_all_mocked=True)153    router.get(ec.URL_COMPANY_TICKERS).respond(json=load_gz("company_tickers.json.gz"))154    router.get(ec.URL_COMPANY_TICKERS_EXCHANGE).respond(json=load_gz("company_tickers_exchange.json.gz"))155    for cik in PROTO_CIKS.values():156        router.get(f"{ec.DATA_BASE}/api/xbrl/companyfacts/CIK{cik:010d}.json").respond(json=load_gz(f"companyfacts_CIK{cik:010d}.json.gz"))157        router.get(f"{ec.DATA_BASE}/submissions/CIK{cik:010d}.json").respond(json=load_gz(f"submissions_CIK{cik:010d}.json.gz"))158    router.get(url__regex=r"https://data\.sec\.gov/api/xbrl/companyfacts/CIK\d+\.json").respond(404)159    router.get(url__regex=r"https://data\.sec\.gov/submissions/CIK\d+\.json").respond(404)160    router.get(url__regex=r"https://www\.sec\.gov/Archives/edgar/data/1620533/000162053326000018/MetaLinks\.json").respond(161        json=load_gz("metalinks_CIK0001620533_000162053326000018.json.gz"))162    router.get(url__regex=r"https://www\.sec\.gov/Archives/edgar/data/.*/MetaLinks\.json").respond(404)163    router.get(url__regex=r"https://www\.sec\.gov/cgi-bin/browse-edgar.*").respond(text=(EDGAR_FIXTURES / "atom_10-Q.xml").read_text())164    router.get(url__regex=r"https://www\.sec\.gov/Archives/edgar/daily-index/.*").respond(text=(EDGAR_FIXTURES / "master.idx").read_text())165    with router:166        yield router167168169@pytest.fixture(scope="session")170def fundamentals_data(app, lake, edgar_mock):171    """Universe + full ingestion of the three prototype companies into the test lake / state DB."""172    from fundamentals import edgar_client, ingest173    client = edgar_client.EdgarClient(rate=10_000, raw_dir=lake / "edgar" / "raw")174    universe = ingest.sync_universe(client)175    results = {tk: ingest.ingest_company(client, cik) for tk, cik in PROTO_CIKS.items()}176    for tk, r in results.items():177        assert r.error is None, f"{tk}: {r.error}"178    ingest._set_state("backfill", companies_total=len(PROTO_CIKS), companies_done=len(PROTO_CIKS))179    ingest._update_mapping_failure_rate()180    return {"universe": universe, "results": results, "client": client}181