SPB Git

Repository Public

e2e_live.py 11.9 KB 289 lines python

1"""Validation de bout en bout contre une instance RÉELLE (vrai modèle), par exemple la production.23    python scripts/e2e_live.py --base https://llm.spboucher.ai --key mlx_sk_…45Utilise le SDK officiel OpenAI sans adaptation, puis des appels HTTP bruts pour ce que le SDK masque6(en-têtes, annulation, erreurs). Chaque contrôle affiche ok/ÉCHEC ; code de sortie ≠ 0 au moindre échec.7"""89from __future__ import annotations1011import argparse12import asyncio13import json14import time15from collections.abc import Callable16from typing import Any1718import httpx19import openai2021MODEL = "maclustr-qwen3"22results: list[tuple[str, bool, str]] = []232425def check(name: str) -> Callable[[Callable[[], Any]], None]:26    def deco(fn: Callable[[], Any]) -> None:27        t0 = time.time()28        try:29            detail = fn()30            results.append((name, True, str(detail or "")))31        except Exception as exc:  # noqa: BLE001 - tout échec est rapporté, aucun n'interrompt la série32            results.append((name, False, f"{type(exc).__name__}: {exc}"[:300]))33        ok, d = results[-1][1], results[-1][2]34        print(f"  {'ok   ' if ok else 'ÉCHEC'} {name}  ({time.time() - t0:.1f} s)  {d}", flush=True)3536    return deco373839def main() -> None:40    ap = argparse.ArgumentParser()41    ap.add_argument("--base", required=True)42    ap.add_argument("--key", required=True)43    args = ap.parse_args()44    base = args.base.rstrip("/")45    client = openai.OpenAI(base_url=f"{base}/v1", api_key=args.key, max_retries=0, timeout=300)46    raw = httpx.Client(base_url=base, headers={"Authorization": f"Bearer {args.key}"}, timeout=300)47    anon = httpx.Client(base_url=base, timeout=30)4849    @check("statut public : opérationnel, modèle annoncé")50    def _() -> str:51        st = anon.get("/api/status").json()52        assert st["status"] == "operational" and st["models"][0]["id"] == MODEL53        assert anon.get("/health").status_code == 200 and anon.get("/ready").status_code == 20054        return f"API {st['api_version']}"5556    @check("pages publiques servies (accueil, docs, playground, statut, OpenAPI)")57    def _() -> str:58        for path in ("/", "/docs", "/docs/streaming", "/playground", "/status", "/openapi.json"):59            r = anon.get(path)60            assert r.status_code == 200, (path, r.status_code)61        assert "MacLustr" in anon.get("/").text62        return ""6364    @check("sans clé : API 401, Dashboard 401")65    def _() -> str:66        assert anon.get("/v1/models").status_code == 40167        assert anon.post("/v1/chat/completions", json={}).status_code == 40168        for path in ("/admin/api/overview", "/admin/api/keys", "/admin/api/audit"):69            assert anon.get(path).status_code == 401, path70        return ""7172    @check("SDK OpenAI : models.list")73    def _() -> str:74        ids = [m.id for m in client.models.list().data]75        assert MODEL in ids76        return ", ".join(ids)7778    @check("SDK OpenAI : réponse non-streaming en français")79    def _() -> str:80        r = client.chat.completions.create(81            model=MODEL,82            max_tokens=60,83            temperature=0,84            messages=[{"role": "user", "content": "Quelle est la capitale du Québec ? Réponds en une phrase."}],85        )86        text = r.choices[0].message.content or ""87        assert "Québec" in text and r.usage and r.usage.completion_tokens > 0 and r.model == MODEL88        return text.strip()[:80]8990    @check("SDK OpenAI : streaming token par token + usage")91    def _() -> str:92        t0 = time.time()93        first = None94        n, usage = 0, None95        stream = client.chat.completions.create(96            model=MODEL,97            max_tokens=120,98            stream=True,99            stream_options={"include_usage": True},100            messages=[{"role": "user", "content": "Compte de 1 à 30, un nombre par ligne."}],101        )102        for chunk in stream:103            if chunk.choices and chunk.choices[0].delta.content:104                first = first or time.time()105                n += 1106            usage = chunk.usage or usage107        assert n > 20 and usage is not None and first is not None108        return f"{n} deltas, premier token {1000 * (first - t0):.0f} ms, {usage.completion_tokens} tokens"109110    @check("conversation multi-tours + message système + cache de préfixe")111    def _() -> str:112        msgs: list[Any] = [113            {"role": "system", "content": "Tu es un assistant qui répond toujours en une seule phrase."},114            {"role": "user", "content": "Mon prénom est Simon-Pierre. Retiens-le."},115        ]116        a = client.chat.completions.create(model=MODEL, max_tokens=60, temperature=0, messages=msgs)117        msgs += [118            {"role": "assistant", "content": a.choices[0].message.content},119            {"role": "user", "content": "Quel est mon prénom ?"},120        ]121        b = client.chat.completions.create(model=MODEL, max_tokens=60, temperature=0, messages=msgs)122        assert "Simon-Pierre" in (b.choices[0].message.content or "")123        cached = (b.usage.prompt_tokens_details.cached_tokens if b.usage and b.usage.prompt_tokens_details else 0) or 0124        assert cached > 0, "aucun token servi depuis le cache de préfixe"125        return f"{cached} tokens de prompt servis depuis le cache"126127    @check("tool calling : appel puis réponse finale")128    def _() -> str:129        tools: Any = [130            {131                "type": "function",132                "function": {133                    "name": "get_weather",134                    "description": "Météo actuelle d'une ville.",135                    "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},136                },137            }138        ]139        msgs: list[Any] = [{"role": "user", "content": "Quel temps fait-il à Gatineau ?"}]140        r = client.chat.completions.create(model=MODEL, max_tokens=200, temperature=0, tools=tools, messages=msgs)141        calls = r.choices[0].message.tool_calls142        assert calls and calls[0].function.name == "get_weather" and r.choices[0].finish_reason == "tool_calls"143        assert "gatineau" in json.loads(calls[0].function.arguments)["city"].lower()144        msgs += [145            r.choices[0].message.model_dump(exclude_none=True),146            {"role": "tool", "tool_call_id": calls[0].id, "content": '{"temp_c": -7, "ciel": "neige"}'},147        ]148        r2 = client.chat.completions.create(model=MODEL, max_tokens=120, temperature=0, tools=tools, messages=msgs)149        text = r2.choices[0].message.content or ""150        assert "7" in text151        return text.strip()[:80]152153    @check("sortie JSON validée par schéma")154    def _() -> str:155        schema = {156            "type": "object",157            "properties": {"ville": {"type": "string"}, "population": {"type": "integer"}},158            "required": ["ville", "population"],159            "additionalProperties": False,160        }161        r = client.chat.completions.create(162            model=MODEL,163            max_tokens=100,164            temperature=0,165            messages=[{"role": "user", "content": "Donne la ville de Montréal et sa population approximative."}],166            response_format={"type": "json_schema", "json_schema": {"name": "ville", "schema": schema}},167        )168        obj = json.loads(r.choices[0].message.content or "")169        assert isinstance(obj["population"], int) and "Montr" in obj["ville"]170        return json.dumps(obj, ensure_ascii=False)171172    @check("paramètres : stop, max_tokens (finish_reason=length), seed reproductible")173    def _() -> str:174        r = client.chat.completions.create(175            model=MODEL, max_tokens=5, messages=[{"role": "user", "content": "Raconte une longue histoire."}]176        )177        assert r.choices[0].finish_reason == "length" and r.usage and r.usage.completion_tokens <= 5178        r = client.chat.completions.create(179            model=MODEL,180            max_tokens=40,181            temperature=0,182            stop=["4"],183            messages=[{"role": "user", "content": "Écris exactement : 1 2 3 4 5 6"}],184        )185        assert "5" not in (r.choices[0].message.content or "")186        a, b = (187            client.chat.completions.create(188                model=MODEL,189                max_tokens=30,190                temperature=1.0,191                seed=1234,192                messages=[{"role": "user", "content": "Invente un nom de planète et décris-la."}],193            )194            .choices[0]195            .message.content196            for _ in range(2)197        )198        assert a == b, "même seed, sorties différentes"199        return ""200201    @check("erreurs typées : modèle inconnu 404, paramètre invalide 400, contexte dépassé 400")202    def _() -> str:203        for exc_type, kw in (204            (openai.NotFoundError, {"model": "gpt-4o"}),205            (openai.BadRequestError, {"model": MODEL, "temperature": 5}),206        ):207            try:208                client.chat.completions.create(messages=[{"role": "user", "content": "x"}], **kw)  # type: ignore[call-overload]209                raise AssertionError(f"{exc_type.__name__} attendu")210            except exc_type:211                pass212        r = raw.post(213            "/v1/chat/completions", json={"model": MODEL, "messages": [{"role": "user", "content": "mot " * 60000}]}214        )215        assert r.status_code == 400 and r.json()["error"]["code"] == "context_length_exceeded"216        return ""217218    @check("en-têtes : x-request-id, limites, sécurité")219    def _() -> str:220        r = raw.post(221            "/v1/chat/completions",222            json={"model": MODEL, "max_tokens": 4, "messages": [{"role": "user", "content": "ok"}]},223        )224        assert r.status_code == 200 and r.headers["x-request-id"] and "x-ratelimit-remaining-requests" in r.headers225        h = anon.get("/").headers226        assert "script-src 'self'" in h["content-security-policy"] and h["x-frame-options"] == "DENY"227        if base.startswith("https://"):228            assert "max-age" in h["strict-transport-security"]229        return ""230231    @check("annulation : fermer le flux libère le moteur")232    def _() -> str:233        with raw.stream(234            "POST",235            "/v1/chat/completions",236            json={237                "model": MODEL,238                "max_tokens": 2000,239                "stream": True,240                "messages": [{"role": "user", "content": "Écris un très long roman."}],241            },242        ) as r:243            it = r.iter_lines()244            for _ in range(12):245                next(it)246        t0 = time.time()247        r2 = raw.post(248            "/v1/chat/completions",249            json={"model": MODEL, "max_tokens": 4, "messages": [{"role": "user", "content": "ok"}]},250        )251        assert r2.status_code == 200 and time.time() - t0 < 15252        return f"requête suivante servie en {time.time() - t0:.1f} s"253254    @check("8 requêtes simultanées : toutes servies")255    def _() -> str:256        async def burst() -> list[int]:257            async with httpx.AsyncClient(258                base_url=base, headers={"Authorization": f"Bearer {args.key}"}, timeout=300259            ) as c:260                rs = await asyncio.gather(261                    *[262                        c.post(263                            "/v1/chat/completions",264                            json={265                                "model": MODEL,266                                "max_tokens": 64,267                                "messages": [{"role": "user", "content": f"Donne un fait sur le nombre {i}."}],268                            },269                        )270                        for i in range(8)271                    ]272                )273                return [r.status_code for r in rs]274275        t0 = time.time()276        codes = asyncio.run(burst())277        assert codes == [200] * 8, codes278        return f"{time.time() - t0:.1f} s"279280    failed = [n for n, ok, _ in results if not ok]281    print(282        f"\n{len(results) - len(failed)}/{len(results)} contrôles réussis" + (f" — ÉCHECS : {failed}" if failed else "")283    )284    raise SystemExit(1 if failed else 0)285286287if __name__ == "__main__":288    main()289
Files main
Menu

Theme