Repository Public
1"""Validation de bout en bout contre une instance RÉELLE (vrai modèle), par exemple la production.23 python scripts/e2e_live.py --base https://llm.spboucher.ai --key mlx_sk_…45Utilise le SDK officiel OpenAI sans adaptation, puis des appels HTTP bruts pour ce que le SDK masque6(en-têtes, annulation, erreurs). Chaque contrôle affiche ok/ÉCHEC ; code de sortie ≠ 0 au moindre échec.7"""89from __future__ import annotations1011import argparse12import asyncio13import json14import time15from collections.abc import Callable16from typing import Any1718import httpx19import openai2021MODEL = "maclustr-qwen3"22results: list[tuple[str, bool, str]] = []232425def check(name: str) -> Callable[[Callable[[], Any]], None]:26 def deco(fn: Callable[[], Any]) -> None:27 t0 = time.time()28 try:29 detail = fn()30 results.append((name, True, str(detail or "")))31 except Exception as exc: # noqa: BLE001 - tout échec est rapporté, aucun n'interrompt la série32 results.append((name, False, f"{type(exc).__name__}: {exc}"[:300]))33 ok, d = results[-1][1], results[-1][2]34 print(f" {'ok ' if ok else 'ÉCHEC'} {name} ({time.time() - t0:.1f} s) {d}", flush=True)3536 return deco373839def main() -> None:40 ap = argparse.ArgumentParser()41 ap.add_argument("--base", required=True)42 ap.add_argument("--key", required=True)43 args = ap.parse_args()44 base = args.base.rstrip("/")45 client = openai.OpenAI(base_url=f"{base}/v1", api_key=args.key, max_retries=0, timeout=300)46 raw = httpx.Client(base_url=base, headers={"Authorization": f"Bearer {args.key}"}, timeout=300)47 anon = httpx.Client(base_url=base, timeout=30)4849 @check("statut public : opérationnel, modèle annoncé")50 def _() -> str:51 st = anon.get("/api/status").json()52 assert st["status"] == "operational" and st["models"][0]["id"] == MODEL53 assert anon.get("/health").status_code == 200 and anon.get("/ready").status_code == 20054 return f"API {st['api_version']}"5556 @check("pages publiques servies (accueil, docs, playground, statut, OpenAPI)")57 def _() -> str:58 for path in ("/", "/docs", "/docs/streaming", "/playground", "/status", "/openapi.json"):59 r = anon.get(path)60 assert r.status_code == 200, (path, r.status_code)61 assert "MacLustr" in anon.get("/").text62 return ""6364 @check("sans clé : API 401, Dashboard 401")65 def _() -> str:66 assert anon.get("/v1/models").status_code == 40167 assert anon.post("/v1/chat/completions", json={}).status_code == 40168 for path in ("/admin/api/overview", "/admin/api/keys", "/admin/api/audit"):69 assert anon.get(path).status_code == 401, path70 return ""7172 @check("SDK OpenAI : models.list")73 def _() -> str:74 ids = [m.id for m in client.models.list().data]75 assert MODEL in ids76 return ", ".join(ids)7778 @check("SDK OpenAI : réponse non-streaming en français")79 def _() -> str:80 r = client.chat.completions.create(81 model=MODEL,82 max_tokens=60,83 temperature=0,84 messages=[{"role": "user", "content": "Quelle est la capitale du Québec ? Réponds en une phrase."}],85 )86 text = r.choices[0].message.content or ""87 assert "Québec" in text and r.usage and r.usage.completion_tokens > 0 and r.model == MODEL88 return text.strip()[:80]8990 @check("SDK OpenAI : streaming token par token + usage")91 def _() -> str:92 t0 = time.time()93 first = None94 n, usage = 0, None95 stream = client.chat.completions.create(96 model=MODEL,97 max_tokens=120,98 stream=True,99 stream_options={"include_usage": True},100 messages=[{"role": "user", "content": "Compte de 1 à 30, un nombre par ligne."}],101 )102 for chunk in stream:103 if chunk.choices and chunk.choices[0].delta.content:104 first = first or time.time()105 n += 1106 usage = chunk.usage or usage107 assert n > 20 and usage is not None and first is not None108 return f"{n} deltas, premier token {1000 * (first - t0):.0f} ms, {usage.completion_tokens} tokens"109110 @check("conversation multi-tours + message système + cache de préfixe")111 def _() -> str:112 msgs: list[Any] = [113 {"role": "system", "content": "Tu es un assistant qui répond toujours en une seule phrase."},114 {"role": "user", "content": "Mon prénom est Simon-Pierre. Retiens-le."},115 ]116 a = client.chat.completions.create(model=MODEL, max_tokens=60, temperature=0, messages=msgs)117 msgs += [118 {"role": "assistant", "content": a.choices[0].message.content},119 {"role": "user", "content": "Quel est mon prénom ?"},120 ]121 b = client.chat.completions.create(model=MODEL, max_tokens=60, temperature=0, messages=msgs)122 assert "Simon-Pierre" in (b.choices[0].message.content or "")123 cached = (b.usage.prompt_tokens_details.cached_tokens if b.usage and b.usage.prompt_tokens_details else 0) or 0124 assert cached > 0, "aucun token servi depuis le cache de préfixe"125 return f"{cached} tokens de prompt servis depuis le cache"126127 @check("tool calling : appel puis réponse finale")128 def _() -> str:129 tools: Any = [130 {131 "type": "function",132 "function": {133 "name": "get_weather",134 "description": "Météo actuelle d'une ville.",135 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},136 },137 }138 ]139 msgs: list[Any] = [{"role": "user", "content": "Quel temps fait-il à Gatineau ?"}]140 r = client.chat.completions.create(model=MODEL, max_tokens=200, temperature=0, tools=tools, messages=msgs)141 calls = r.choices[0].message.tool_calls142 assert calls and calls[0].function.name == "get_weather" and r.choices[0].finish_reason == "tool_calls"143 assert "gatineau" in json.loads(calls[0].function.arguments)["city"].lower()144 msgs += [145 r.choices[0].message.model_dump(exclude_none=True),146 {"role": "tool", "tool_call_id": calls[0].id, "content": '{"temp_c": -7, "ciel": "neige"}'},147 ]148 r2 = client.chat.completions.create(model=MODEL, max_tokens=120, temperature=0, tools=tools, messages=msgs)149 text = r2.choices[0].message.content or ""150 assert "7" in text151 return text.strip()[:80]152153 @check("sortie JSON validée par schéma")154 def _() -> str:155 schema = {156 "type": "object",157 "properties": {"ville": {"type": "string"}, "population": {"type": "integer"}},158 "required": ["ville", "population"],159 "additionalProperties": False,160 }161 r = client.chat.completions.create(162 model=MODEL,163 max_tokens=100,164 temperature=0,165 messages=[{"role": "user", "content": "Donne la ville de Montréal et sa population approximative."}],166 response_format={"type": "json_schema", "json_schema": {"name": "ville", "schema": schema}},167 )168 obj = json.loads(r.choices[0].message.content or "")169 assert isinstance(obj["population"], int) and "Montr" in obj["ville"]170 return json.dumps(obj, ensure_ascii=False)171172 @check("paramètres : stop, max_tokens (finish_reason=length), seed reproductible")173 def _() -> str:174 r = client.chat.completions.create(175 model=MODEL, max_tokens=5, messages=[{"role": "user", "content": "Raconte une longue histoire."}]176 )177 assert r.choices[0].finish_reason == "length" and r.usage and r.usage.completion_tokens <= 5178 r = client.chat.completions.create(179 model=MODEL,180 max_tokens=40,181 temperature=0,182 stop=["4"],183 messages=[{"role": "user", "content": "Écris exactement : 1 2 3 4 5 6"}],184 )185 assert "5" not in (r.choices[0].message.content or "")186 a, b = (187 client.chat.completions.create(188 model=MODEL,189 max_tokens=30,190 temperature=1.0,191 seed=1234,192 messages=[{"role": "user", "content": "Invente un nom de planète et décris-la."}],193 )194 .choices[0]195 .message.content196 for _ in range(2)197 )198 assert a == b, "même seed, sorties différentes"199 return ""200201 @check("erreurs typées : modèle inconnu 404, paramètre invalide 400, contexte dépassé 400")202 def _() -> str:203 for exc_type, kw in (204 (openai.NotFoundError, {"model": "gpt-4o"}),205 (openai.BadRequestError, {"model": MODEL, "temperature": 5}),206 ):207 try:208 client.chat.completions.create(messages=[{"role": "user", "content": "x"}], **kw) # type: ignore[call-overload]209 raise AssertionError(f"{exc_type.__name__} attendu")210 except exc_type:211 pass212 r = raw.post(213 "/v1/chat/completions", json={"model": MODEL, "messages": [{"role": "user", "content": "mot " * 60000}]}214 )215 assert r.status_code == 400 and r.json()["error"]["code"] == "context_length_exceeded"216 return ""217218 @check("en-têtes : x-request-id, limites, sécurité")219 def _() -> str:220 r = raw.post(221 "/v1/chat/completions",222 json={"model": MODEL, "max_tokens": 4, "messages": [{"role": "user", "content": "ok"}]},223 )224 assert r.status_code == 200 and r.headers["x-request-id"] and "x-ratelimit-remaining-requests" in r.headers225 h = anon.get("/").headers226 assert "script-src 'self'" in h["content-security-policy"] and h["x-frame-options"] == "DENY"227 if base.startswith("https://"):228 assert "max-age" in h["strict-transport-security"]229 return ""230231 @check("annulation : fermer le flux libère le moteur")232 def _() -> str:233 with raw.stream(234 "POST",235 "/v1/chat/completions",236 json={237 "model": MODEL,238 "max_tokens": 2000,239 "stream": True,240 "messages": [{"role": "user", "content": "Écris un très long roman."}],241 },242 ) as r:243 it = r.iter_lines()244 for _ in range(12):245 next(it)246 t0 = time.time()247 r2 = raw.post(248 "/v1/chat/completions",249 json={"model": MODEL, "max_tokens": 4, "messages": [{"role": "user", "content": "ok"}]},250 )251 assert r2.status_code == 200 and time.time() - t0 < 15252 return f"requête suivante servie en {time.time() - t0:.1f} s"253254 @check("8 requêtes simultanées : toutes servies")255 def _() -> str:256 async def burst() -> list[int]:257 async with httpx.AsyncClient(258 base_url=base, headers={"Authorization": f"Bearer {args.key}"}, timeout=300259 ) as c:260 rs = await asyncio.gather(261 *[262 c.post(263 "/v1/chat/completions",264 json={265 "model": MODEL,266 "max_tokens": 64,267 "messages": [{"role": "user", "content": f"Donne un fait sur le nombre {i}."}],268 },269 )270 for i in range(8)271 ]272 )273 return [r.status_code for r in rs]274275 t0 = time.time()276 codes = asyncio.run(burst())277 assert codes == [200] * 8, codes278 return f"{time.time() - t0:.1f} s"279280 failed = [n for n, ok, _ in results if not ok]281 print(282 f"\n{len(results) - len(failed)}/{len(results)} contrôles réussis" + (f" — ÉCHECS : {failed}" if failed else "")283 )284 raise SystemExit(1 if failed else 0)285286287if __name__ == "__main__":288 main()289