SPB Git

Repository Public

test_api.py 14.8 KB 312 lines python

1"""API compatible OpenAI : authentification, formats, streaming, outils, JSON, erreurs, SDK officiel."""23from __future__ import annotations45import json6import time7from typing import Any89import fake_engine10import httpx11import pytest12from conftest import MODEL, Stack131415def chat(c: httpx.Client, content: str = "bonjour", **extra: Any) -> httpx.Response:16    return c.post(17        "/v1/chat/completions", json={"model": MODEL, "messages": [{"role": "user", "content": content}], **extra}18    )192021def sse(resp: httpx.Response) -> list[Any]:22    out: list[Any] = []23    for line in resp.text.splitlines():24        if line.startswith("data: "):25            out.append(line[6:] if line[6:] == "[DONE]" else json.loads(line[6:]))26    return out272829def test_health_ready_status_are_public(stack: Stack) -> None:30    with stack.client() as c:31        assert c.get("/health").json() == {"status": "ok"}32        assert c.get("/ready").status_code == 20033        st = c.get("/api/status").json()34        assert st["status"] == "operational" and st["models"][0]["available"] is True35        # le statut public n'expose aucune métrique interne36        assert set(st) == {"status", "api_version", "models", "engine", "last_updated", "checked_at"}373839def test_auth_is_checked_before_body(stack: Stack) -> None:40    with stack.client() as c:41        r = c.post("/v1/chat/completions", json={})42        assert r.status_code == 401 and r.json()["error"]["code"] == "missing_api_key"43        assert r.headers["www-authenticate"] == "Bearer"44        r = c.get("/v1/models", headers={"Authorization": "Bearer mlx_sk_deadbeef_nope"})45        assert r.status_code == 401 and r.json()["error"]["code"] == "invalid_api_key"464748def test_models(stack: Stack) -> None:49    _, key = stack.make_key(models=[MODEL])50    with stack.client(key) as c:51        data = c.get("/v1/models").json()["data"]52        assert [m["id"] for m in data] == [MODEL]  # le modèle non autorisé est invisible53        assert c.get(f"/v1/models/{MODEL}").json()["context_window"] == 204854        assert c.get("/v1/models/other-model").status_code == 40455        r = c.post(56            "/v1/chat/completions", json={"model": "other-model", "messages": [{"role": "user", "content": "x"}]}57        )58        assert r.status_code == 403 and r.json()["error"]["code"] == "model_not_allowed"59        r = chat(c, model="nope")60        assert r.status_code == 404 and r.json()["error"]["code"] == "model_not_found"616263def test_chat_non_stream(stack: Stack) -> None:64    _, key = stack.make_key()65    with stack.client(key) as c:66        r = chat(c, "salut")67        assert r.status_code == 20068        body = r.json()69        assert body["object"] == "chat.completion" and body["model"] == MODEL70        assert body["id"].startswith("chatcmpl-") and body["id"].endswith(r.headers["x-request-id"])71        assert body["choices"][0]["message"] == {"role": "assistant", "content": "echo: salut "}72        assert body["choices"][0]["finish_reason"] == "stop"73        u = body["usage"]74        assert u["total_tokens"] == u["prompt_tokens"] + u["completion_tokens"] > 075        assert u["prompt_tokens_details"]["cached_tokens"] == 276        assert r.headers["x-ratelimit-limit-requests"] == "60"77    # le moteur reçoit toujours « default_model » : un client ne peut pas faire charger un autre modèle78    assert fake_engine.STATE["bodies"][-1]["model"] == "default_model"798081def test_chat_stream_format(stack: Stack) -> None:82    _, key = stack.make_key()83    with stack.client(key) as c:84        r = chat(c, "a b c", stream=True, stream_options={"include_usage": True})85        assert r.headers["content-type"].startswith("text/event-stream")86        events = sse(r)87        assert events[-1] == "[DONE]"88        chunks = events[:-1]89        assert chunks[0]["choices"][0]["delta"] == {"role": "assistant", "content": ""}90        text = "".join(ch["choices"][0]["delta"].get("content", "") for ch in chunks if ch["choices"])91        assert text == "echo: a b c "92        assert chunks[-2]["choices"][0]["finish_reason"] == "stop"93        assert chunks[-1]["choices"] == [] and chunks[-1]["usage"]["completion_tokens"] == 494        assert all(ch["object"] == "chat.completion.chunk" and ch["model"] == MODEL for ch in chunks)95        assert ": keepalive" in r.text96        # sans include_usage, pas de bloc d'usage97        events = sse(chat(c, "a", stream=True))98        assert all("usage" not in e for e in events[:-1])99100101def test_tool_calling(stack: Stack) -> None:102    _, key = stack.make_key()103    tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}]104    with stack.client(key) as c:105        body = chat(c, "météo [tool]", tools=tools).json()106        msg = body["choices"][0]["message"]107        assert body["choices"][0]["finish_reason"] == "tool_calls" and msg["content"] is None108        call = msg["tool_calls"][0]109        assert call["type"] == "function" and call["function"]["name"] == "get_weather"110        assert json.loads(call["function"]["arguments"]) == {"city": "Québec"} and call["id"].startswith("call_")111        # tour suivant : l'historique avec tool_calls + rôle tool est accepté et transmis au moteur112        r = c.post(113            "/v1/chat/completions",114            json={115                "model": MODEL,116                "tools": tools,117                "messages": [118                    {"role": "user", "content": "météo"},119                    {"role": "assistant", "content": None, "tool_calls": [call]},120                    {"role": "tool", "tool_call_id": call["id"], "content": "-4 °C"},121                    {"role": "user", "content": "merci"},122                ],123            },124        )125        assert r.status_code == 200126        sent = fake_engine.STATE["bodies"][-1]["messages"]127        assert sent[1]["tool_calls"][0]["function"]["arguments"] == {"city": "Québec"}128        assert sent[2] == {"role": "tool", "content": "-4 °C", "tool_call_id": call["id"]}129        # tool_choice130        assert "tools" not in (chat(c, "x [tool]", tools=tools, tool_choice="none"), fake_engine.STATE["bodies"][-1])[1]131        r = chat(c, "x", tools=tools, tool_choice="required")132        assert r.status_code == 400 and r.json()["error"]["code"] == "unsupported_parameter"133134135def test_reasoning_and_thinking_flag(stack: Stack) -> None:136    _, key = stack.make_key()137    with stack.client(key) as c:138        body = chat(c, "[think] ok", enable_thinking=True).json()139        assert body["choices"][0]["message"]["reasoning_content"] == "je réfléchis"140        assert fake_engine.STATE["bodies"][-1]["chat_template_kwargs"] == {"enable_thinking": True}141        chat(c, "ok")142        assert fake_engine.STATE["bodies"][-1]["chat_template_kwargs"] == {"enable_thinking": False}143144145def test_json_mode_validated(stack: Stack) -> None:146    _, key = stack.make_key()147    schema = {"type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"]}148    fmt = {"type": "json_schema", "json_schema": {"name": "r", "schema": schema}}149    with stack.client(key) as c:150        body = chat(c, "[json]", response_format=fmt).json()151        assert json.loads(body["choices"][0]["message"]["content"]) == {"answer": 42}152        assert "JSON Schema" in fake_engine.STATE["bodies"][-1]["messages"][0]["content"]153        # sortie invalide au premier essai → un essai de réparation, facturé, puis succès154        n = fake_engine.STATE["requests"]155        body = chat(c, "[badjson]", response_format={"type": "json_object"}).json()156        assert json.loads(body["choices"][0]["message"]["content"]) == {"answer": 42}157        assert fake_engine.STATE["requests"] == n + 2 and body["usage"]["completion_tokens"] == 4158        # schéma impossible à satisfaire → 422 explicite plutôt qu'un JSON faux159        bad = {160            "type": "json_schema",161            "json_schema": {"name": "r", "schema": {"type": "object", "required": ["missing"]}},162        }163        r = chat(c, "[json]", response_format=bad)164        assert r.status_code == 422 and r.json()["error"]["code"] == "json_validation_failed"165        r = chat(c, "x", response_format={"type": "json_schema", "json_schema": {"name": "r", "schema": {"type": 3}}})166        assert r.status_code == 400167168169@pytest.mark.parametrize(170    ("patch", "param"),171    [172        ({"n": 2}, "n"),173        ({"temperature": 3}, "temperature"),174        ({"top_p": 0}, "top_p"),175        ({"max_tokens": 0}, "max_tokens"),176        ({"messages": []}, "messages"),177        ({"stop": ["a"] * 9}, "stop"),178        (179            {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "x"}}]}]},180            "messages.0.content",181        ),182    ],183)184def test_validation_errors(stack: Stack, patch: dict[str, Any], param: str) -> None:185    _, key = stack.make_key()186    with stack.client(key) as c:187        r = c.post(188            "/v1/chat/completions", json={"model": MODEL, "messages": [{"role": "user", "content": "x"}], **patch}189        )190        assert r.status_code == 400, r.text191        err = r.json()["error"]192        assert err["type"] == "invalid_request_error" and err["param"] == param and err["request_id"]193194195def test_context_and_size_limits(stack: Stack) -> None:196    _, key = stack.make_key(tpm=0)197    with stack.client(key) as c:198        r = chat(c, "mot " * 9000)199        assert r.status_code == 400 and r.json()["error"]["code"] == "context_length_exceeded"200        r = c.post("/v1/chat/completions", content=b"x" * 300_000, headers={"content-type": "application/json"})201        assert r.status_code == 413 and r.json()["error"]["code"] == "request_too_large"202        # max_tokens est borné par le modèle (512) quel que soit le souhait du client203        chat(c, "x", max_tokens=100000)204        assert fake_engine.STATE["bodies"][-1]["max_tokens"] == 512205206207def test_text_completions(stack: Stack) -> None:208    _, key = stack.make_key()209    with stack.client(key) as c:210        body = c.post("/v1/completions", json={"model": MODEL, "prompt": "il était"}).json()211        assert body["object"] == "text_completion" and body["choices"][0]["text"] == "echo: il était "212213214def test_endpoint_permission(stack: Stack) -> None:215    _, key = stack.make_key(endpoints=["models"])216    with stack.client(key) as c:217        assert c.get("/v1/models").status_code == 200218        r = chat(c)219        assert r.status_code == 403 and r.json()["error"]["code"] == "endpoint_not_allowed"220221222def test_revoked_disabled_expired_keys(stack: Stack) -> None:223    k1, s1 = stack.make_key()224    k2, s2 = stack.make_key()225    _, s3 = stack.make_key(expires_at=int(time.time()) + 1)226    with stack.client() as c:227228        def ok(secret: str) -> int:229            return c.get("/v1/models", headers={"Authorization": f"Bearer {secret}"}).status_code230231        assert ok(s1) == ok(s2) == ok(s3) == 200232        stack.run(stack.svc.keys.revoke(k1["id"]))233        stack.run(stack.svc.keys.update(k2["id"], {"disabled": True}))234        time.sleep(1.1)235        assert ok(s1) == ok(s2) == ok(s3) == 401236237238def test_engine_failure_mid_stream_and_down(stack: Stack) -> None:239    _, key = stack.make_key()240    with stack.client(key) as c:241        events = sse(chat(c, "a b [boom]", stream=True))242        assert "error" in events[-1] and events[-1]["error"]["code"] == "engine_unavailable"243        r = chat(c, "a b [boom]")244        assert r.status_code == 503245        # moteur arrêté : la passerelle reste debout et répond 503 proprement246        stack.engine_server.stop()247        stack.run(stack.svc.engine.check_all())248        deadline = time.time() + 10249        while time.time() < deadline and c.get("/ready").status_code != 503:250            time.sleep(0.2)251        assert c.get("/health").status_code == 200 and c.get("/ready").status_code == 503252        r = chat(c)253        assert r.status_code == 503 and r.json()["error"]["code"] == "engine_unavailable"254        assert c.get("/api/status").json()["status"] == "unavailable"255256257def test_cors_and_security_headers(stack: Stack) -> None:258    with stack.client() as c:259        r = c.options(260            "/v1/chat/completions", headers={"Origin": "https://app.example", "Access-Control-Request-Method": "POST"}261        )262        assert r.status_code == 204 and r.headers["access-control-allow-origin"] == "*"263        assert "access-control-allow-credentials" not in r.headers264        r = c.get("/")265        assert "script-src 'self'" in r.headers["content-security-policy"]266        assert r.headers["x-frame-options"] == "DENY" and r.headers["x-content-type-options"] == "nosniff"267        assert c.get("/admin").headers["cache-control"] == "no-store"268269270def test_openapi_contract_matches_backend(stack: Stack) -> None:271    with stack.client() as c:272        spec = c.get("/openapi.json").json()273    props = spec["components"]["schemas"]["ChatRequest"]["properties"]274    for field in ("model", "messages", "stream", "temperature", "top_p", "tools", "response_format", "max_tokens"):275        assert field in props276    assert spec["paths"]["/v1/chat/completions"]["post"]["security"] == [{"bearerAuth": []}]277    assert spec["x-error-codes"]["rate_limit_exceeded"]["status"] == 429278    assert "/admin/api/keys" not in spec["paths"]279280281def test_official_openai_sdk(stack: Stack) -> None:282    """Le SDK officiel OpenAI, non modifié, fonctionne : non-streaming, streaming, outils, erreurs typées."""283    import openai284285    _, key = stack.make_key()286    client = openai.OpenAI(base_url=f"{stack.base}/v1", api_key=key, max_retries=0)287    assert [m.id for m in client.models.list().data] == [MODEL, "other-model"]288    resp = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "allô"}])289    assert resp.choices[0].message.content == "echo: allô " and resp.usage and resp.usage.total_tokens > 0290    stream = client.chat.completions.create(291        model=MODEL,292        messages=[{"role": "user", "content": "un deux"}],293        stream=True,294        stream_options={"include_usage": True},295    )296    text, usage = "", None297    for chunk in stream:298        if chunk.choices and chunk.choices[0].delta.content:299            text += chunk.choices[0].delta.content300        usage = chunk.usage or usage301    assert text == "echo: un deux " and usage is not None and usage.completion_tokens == 3302    tools: Any = [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}]303    resp = client.chat.completions.create(model=MODEL, tools=tools, messages=[{"role": "user", "content": "[tool]"}])304    calls = resp.choices[0].message.tool_calls305    assert calls and calls[0].function.name == "get_weather"306    with pytest.raises(openai.AuthenticationError):307        openai.OpenAI(base_url=f"{stack.base}/v1", api_key="mlx_sk_bad_bad", max_retries=0).models.list()308    with pytest.raises(openai.NotFoundError):309        client.chat.completions.create(model="nope", messages=[{"role": "user", "content": "x"}])310    with pytest.raises(openai.BadRequestError):311        client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "x"}], temperature=9)312
Files main
Menu

Theme