Repository Public
1"""Banc d'essai reproductible de MacLustr LLM API.23Mesure, contre un moteur MLX (ou la passerelle), les scénarios du cahier des charges : prompt court, prompt4long, conversation multi-tours, génération longue, programmation, raisonnement, tool calling, préfixes répétés,5contexte volumineux et montée en concurrence (1, 2, 4, 8, 16 clients).67 python bench/bench.py --base http://127.0.0.1:8811 --stats http://127.0.0.1:8811/internal/stats \8 --label instruct-2507-4bit --out ~/maclustr-llm-data/bench910Chaque valeur du rapport est mesurée pendant l'exécution ; rien n'est estimé. Les temps sont pris côté client :11TTFT = délai entre l'envoi et le premier delta de contenu ; tokens/s = (tokens − 1) / (dernier − premier delta).12"""1314from __future__ import annotations1516import argparse17import asyncio18import json19import platform20import statistics21import subprocess22import time23from dataclasses import asdict, dataclass, field24from pathlib import Path25from typing import Any2627import httpx2829FILLER = (30 "Le calcul distribué sur Apple Silicon repose sur la mémoire unifiée : le processeur central et le "31 "processeur graphique partagent les mêmes pages, ce qui évite les copies entre mémoire système et mémoire "32 "vidéo. Un modèle de langage quantifié sur quatre bits y tient donc en entier, et le débit dépend surtout "33 "de la bande passante mémoire. "34)353637def long_text(approx_tokens: int, salt: str = "") -> str:38 """Texte français d'environ `approx_tokens` tokens (≈ 120 tokens mesurés par phrase de remplissage)."""39 n = max(1, approx_tokens // 120)40 return " ".join(f"[{salt}{i}] {FILLER}" for i in range(n))414243@dataclass44class Sample:45 ok: bool46 ttft_ms: float | None = None47 latency_ms: float = 0.048 gen_ms: float = 0.049 prompt_tokens: int = 050 completion_tokens: int = 051 cached_tokens: int = 052 finish_reason: str | None = None53 tool_calls: int = 054 error: str | None = None55 text: str = field(default="", repr=False)5657 @property58 def tps(self) -> float | None:59 if self.completion_tokens > 1 and self.gen_ms > 0:60 return (self.completion_tokens - 1) / (self.gen_ms / 1000)61 return None626364class Client:65 def __init__(self, base: str, model: str, key: str | None) -> None:66 self.base = base.rstrip("/")67 self.model = model68 headers = {"Authorization": f"Bearer {key}"} if key else {}69 self.http = httpx.AsyncClient(70 timeout=httpx.Timeout(900, connect=5), headers=headers, limits=httpx.Limits(max_connections=64)71 )7273 async def chat(self, messages: list[dict[str, Any]], max_tokens: int = 256, **extra: Any) -> Sample:74 payload = {75 "model": self.model,76 "messages": messages,77 "max_tokens": max_tokens,78 "stream": True,79 "stream_options": {"include_usage": True},80 "temperature": 0.0,81 **extra,82 }83 s = Sample(ok=False)84 t0 = time.perf_counter()85 first = last = None86 try:87 async with self.http.stream("POST", f"{self.base}/v1/chat/completions", json=payload) as r:88 if r.status_code != 200:89 s.error = f"HTTP {r.status_code}: {(await r.aread())[:200]!r}"90 return s91 async for line in r.aiter_lines():92 if not line.startswith("data:"):93 continue94 data = line[5:].strip()95 if data == "[DONE]":96 break97 evt = json.loads(data)98 if "error" in evt:99 s.error = str(evt["error"])[:200]100 return s101 if evt.get("usage"):102 u = evt["usage"]103 s.prompt_tokens, s.completion_tokens = u["prompt_tokens"], u["completion_tokens"]104 s.cached_tokens = (u.get("prompt_tokens_details") or {}).get("cached_tokens", 0)105 for ch in evt.get("choices") or []:106 d = ch.get("delta") or {}107 piece = d.get("content") or d.get("reasoning") or d.get("reasoning_content")108 if piece or d.get("tool_calls"):109 now = time.perf_counter()110 first = first or now111 last = now112 s.text += d.get("content") or ""113 s.tool_calls += len(d.get("tool_calls") or [])114 if ch.get("finish_reason"):115 s.finish_reason = ch["finish_reason"]116 s.ok = True117 except Exception as exc: # noqa: BLE001 - toute panne compte comme une erreur mesurée118 s.error = f"{type(exc).__name__}: {exc}"[:200]119 s.latency_ms = (time.perf_counter() - t0) * 1000120 if first is not None and last is not None:121 s.ttft_ms = (first - t0) * 1000122 s.gen_ms = (last - first) * 1000123 return s124125126def pct(vals: list[float], p: float) -> float | None:127 if not vals:128 return None129 vals = sorted(vals)130 return round(vals[min(len(vals) - 1, int(round(p * (len(vals) - 1))))], 1)131132133def summarize(name: str, description: str, samples: list[Sample], wall_s: float | None = None) -> dict[str, Any]:134 ok = [s for s in samples if s.ok]135 ttft = [s.ttft_ms for s in ok if s.ttft_ms is not None]136 lat = [s.latency_ms for s in ok]137 tps = [t for s in ok if (t := s.tps) is not None]138 out: dict[str, Any] = {139 "name": name,140 "description": description,141 "requests": len(samples),142 "error_rate": round(1 - len(ok) / len(samples), 4) if samples else None,143 "ttft_ms_p50": pct(ttft, 0.5),144 "ttft_ms_p95": pct(ttft, 0.95),145 "ttft_ms_p99": pct(ttft, 0.99),146 "latency_ms_p50": pct(lat, 0.5),147 "latency_ms_p95": pct(lat, 0.95),148 "latency_ms_p99": pct(lat, 0.99),149 "tokens_per_second_p50": pct(tps, 0.5),150 "prompt_tokens_avg": round(statistics.mean(s.prompt_tokens for s in ok)) if ok else None,151 "completion_tokens_avg": round(statistics.mean(s.completion_tokens for s in ok)) if ok else None,152 "cached_tokens_avg": round(statistics.mean(s.cached_tokens for s in ok)) if ok else None,153 "cache_hit_ratio": round(sum(s.cached_tokens for s in ok) / max(1, sum(s.prompt_tokens for s in ok)), 3)154 if ok155 else None,156 "errors": sorted({s.error for s in samples if s.error})[:3],157 }158 if wall_s:159 out["wall_s"] = round(wall_s, 2)160 out["aggregate_tps"] = round(sum(s.completion_tokens for s in ok) / wall_s, 1)161 return out162163164def sh(cmd: list[str]) -> str:165 try:166 return subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=False).stdout.strip()167 except (OSError, subprocess.SubprocessError):168 return ""169170171def memory_state() -> dict[str, Any]:172 level = sh(["/usr/sbin/sysctl", "-n", "kern.memorystatus_vm_pressure_level"])173 free = sh(["/usr/sbin/sysctl", "-n", "kern.memorystatus_level"])174 swap = sh(["/usr/sbin/sysctl", "-n", "vm.swapusage"])175 return {176 "pressure_level": {"1": "normal", "2": "warn", "4": "critical"}.get(level, level),177 "free_percent": int(free) if free.isdigit() else None,178 "swap": swap,179 }180181182WEATHER_TOOL = [183 {184 "type": "function",185 "function": {186 "name": "get_weather",187 "description": "Retourne la météo actuelle d'une ville.",188 "parameters": {189 "type": "object",190 "properties": {191 "city": {"type": "string"},192 "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},193 },194 "required": ["city"],195 },196 },197 }198]199200201async def run(args: argparse.Namespace) -> dict[str, Any]:202 c = Client(args.base, args.model, args.key)203 stats_http = httpx.AsyncClient(timeout=5)204205 async def engine_stats() -> dict[str, Any] | None:206 if not args.stats:207 return None208 try:209 return (await stats_http.get(args.stats)).json()210 except Exception: # noqa: BLE001211 return None212213 scenarios: list[dict[str, Any]] = []214 extra = json.loads(args.extra) if args.extra else {}215216 async def seq(name: str, desc: str, build: Any, n: int, max_tokens: int, **kw: Any) -> list[Sample]:217 samples = [await c.chat(build(i), max_tokens=max_tokens, **extra, **kw) for i in range(n)]218 scenarios.append(summarize(name, desc, samples))219 print(json.dumps(scenarios[-1], ensure_ascii=False), flush=True)220 return samples221222 # réchauffage (hors mesure)223 await c.chat([{"role": "user", "content": "Bonjour."}], max_tokens=16, **extra)224 start_stats = await engine_stats()225226 single = await seq(227 "short_prompt",228 "Question courte, réponse de 128 tokens.",229 lambda i: [230 {231 "role": "user",232 "content": f"Explique en quelques phrases le théorème n° {i + 1} de ton choix en mathématiques.",233 }234 ],235 args.n,236 128,237 )238 if not args.quick:239 await seq(240 "long_generation",241 "Génération longue de 1 024 tokens.",242 lambda i: [243 {244 "role": "user",245 "content": f"Rédige un essai détaillé (variante {i}) sur l'histoire de "246 "l'informatique, des origines à aujourd'hui.",247 }248 ],249 3,250 1024,251 )252 await seq(253 "long_prompt",254 "Prompt d'environ 4 000 tokens (jamais vu), réponse de 128 tokens.",255 lambda i: [{"role": "user", "content": long_text(4000, f"L{i}-") + "\n\nRésume ce texte en 3 phrases."}],256 3,257 128,258 )259 await seq(260 "large_context",261 "Contexte d'environ 16 000 tokens (jamais vu), réponse de 128 tokens.",262 lambda i: [{"role": "user", "content": long_text(16000, f"X{i}-") + "\n\nRésume ce texte en 3 phrases."}],263 2,264 128,265 )266 await seq(267 "coding",268 "Programmation : fonction Python avec tests.",269 lambda i: [270 {271 "role": "user",272 "content": "Écris une fonction Python `merge_intervals(intervals)` qui "273 "fusionne des intervalles qui se chevauchent, avec docstring "274 f"et 3 tests. Variante {i}.",275 }276 ],277 3,278 512,279 )280 await seq(281 "reasoning",282 "Raisonnement en plusieurs étapes.",283 lambda i: [284 {285 "role": "user",286 "content": f"Un train part à {8 + i} h à 90 km/h, un second part une heure "287 "plus tard du même point à 120 km/h. À quelle heure le second "288 "rattrape-t-il le premier ? Détaille le raisonnement.",289 }290 ],291 3,292 400,293 )294 tool_samples = await seq(295 "tool_calling",296 "Appel d'outil (get_weather) attendu.",297 lambda i: [298 {299 "role": "user",300 "content": [301 "Quel temps fait-il à Québec ?",302 "Donne-moi la météo de Montréal en celsius.",303 "Météo à Paris ?",304 ][i % 3],305 }306 ],307 3,308 200,309 tools=WEATHER_TOOL,310 )311 scenarios[-1]["tool_call_rate"] = round(sum(1 for s in tool_samples if s.tool_calls > 0) / len(tool_samples), 2)312313 # conversation multi-tours : chaque tour réutilise le préfixe du précédent (cache KV)314 history: list[dict[str, Any]] = [{"role": "system", "content": "Tu es un assistant concis."}]315 turns: list[Sample] = []316 for q in [317 "Présente Montréal en 4 phrases.",318 "Et son climat ?",319 "Quels quartiers visiter ?",320 "Compare avec Québec.",321 "Résume notre échange en 3 points.",322 ]:323 history.append({"role": "user", "content": q})324 s = await c.chat(history, max_tokens=160, **extra)325 turns.append(s)326 history.append({"role": "assistant", "content": s.text})327 scenarios.append(summarize("multi_turn", "Conversation de 5 tours (préfixe réutilisé d'un tour à l'autre).", turns))328 print(json.dumps(scenarios[-1], ensure_ascii=False), flush=True)329330 # préfixes répétés : même prompt système d'environ 3 000 tokens, questions différentes331 system = "Tu es l'assistant d'une documentation technique. Contexte :\n" + long_text(3000, "DOC-")332 cold = await c.chat(333 [334 {"role": "system", "content": system},335 {"role": "user", "content": "Question 0 : de quoi parle ce contexte ?"},336 ],337 max_tokens=64,338 **extra,339 )340 warm = [341 await c.chat(342 [343 {"role": "system", "content": system},344 {"role": "user", "content": f"Question {i} : cite un mot-clé du contexte."},345 ],346 max_tokens=64,347 **extra,348 )349 for i in range(1, 6)350 ]351 scenarios.append(352 summarize(353 "repeated_prefix",354 "Préfixe système commun d'environ 3 000 tokens, 5 questions différentes après un premier appel à froid.",355 warm,356 )357 )358 prefix_cache = {359 "cold_ttft_ms": round(cold.ttft_ms, 1) if cold.ttft_ms else None,360 "warm_ttft_ms": pct([s.ttft_ms for s in warm if s.ttft_ms], 0.5),361 "warm_cached_tokens": warm[-1].cached_tokens,362 "prompt_tokens": warm[-1].prompt_tokens,363 }364 print(json.dumps({**scenarios[-1], "prefix_cache": prefix_cache}, ensure_ascii=False), flush=True)365366 # montée en concurrence367 concurrency: list[dict[str, Any]] = []368 for n in [int(x) for x in args.concurrency.split(",")]:369 before = await engine_stats()370 mem_before = memory_state()371 t0 = time.perf_counter()372 samples: list[Sample] = []373 for r in range(args.rounds): # chaque manche lance exactement n générations simultanées374 samples += list(375 await asyncio.gather(376 *[377 c.chat(378 [379 {380 "role": "user",381 "content": f"Rédige un texte informatif (sujet n° {r * 100 + i}) "382 "d'environ 300 mots sur une invention scientifique de ton choix.",383 }384 ],385 max_tokens=args.concurrency_tokens,386 **extra,387 )388 for i in range(n)389 ]390 )391 )392 wall = time.perf_counter() - t0393 after = await engine_stats()394 s = summarize(395 f"concurrency_{n}", f"{n} générations simultanées de {args.concurrency_tokens} tokens.", samples, wall396 )397 row = {398 "clients": n,399 "aggregate_tps": s["aggregate_tps"],400 "per_request_tps": s["tokens_per_second_p50"],401 "ttft_p50_ms": s["ttft_ms_p50"],402 "ttft_p95_ms": s["ttft_ms_p95"],403 "latency_p50_ms": s["latency_ms_p50"],404 "latency_p95_ms": s["latency_ms_p95"],405 "latency_p99_ms": s["latency_ms_p99"],406 "error_rate": s["error_rate"],407 "wall_s": s["wall_s"],408 "mlx_peak_bytes": (after or {}).get("mlx", {}).get("peak_bytes"),409 "mlx_active_bytes": (after or {}).get("mlx", {}).get("active_bytes"),410 "memory_before": mem_before,411 "memory_after": memory_state(),412 "errors": s["errors"],413 }414 del before415 concurrency.append(row)416 print(json.dumps(row, ensure_ascii=False), flush=True)417418 end_stats = await engine_stats()419 ok_rows = [r for r in concurrency if r["error_rate"] == 0 and r["memory_after"]["pressure_level"] == "normal"]420 best = max(ok_rows, key=lambda r: r["aggregate_tps"]) if ok_rows else None421 single_s = summarize("single", "", single)422 chip = sh(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"])423 mem = sh(["/usr/sbin/sysctl", "-n", "hw.memsize"])424 report = {425 "meta": {426 "date": time.strftime("%Y-%m-%d %H:%M:%S"),427 "label": args.label,428 "host": platform.node(),429 "chip": chip,430 "memory_gb": round(int(mem) / (1 << 30)) if mem.isdigit() else None,431 "macos": platform.mac_ver()[0],432 "mlx": (end_stats or {}).get("versions", {}).get("mlx"),433 "mlx_lm": (end_stats or {}).get("versions", {}).get("mlx_lm"),434 "model": (end_stats or {}).get("model"),435 "base": args.base,436 "extra": extra,437 "decode_concurrency": (end_stats or {}).get("decode_concurrency"),438 "prompt_concurrency": (end_stats or {}).get("prompt_concurrency"),439 },440 "summary": {441 "model": (end_stats or {}).get("model") or args.label,442 "single": {"ttft_ms": single_s["ttft_ms_p50"], "tokens_per_second": single_s["tokens_per_second_p50"]},443 "concurrency": [444 {445 k: r[k]446 for k in (447 "clients",448 "aggregate_tps",449 "per_request_tps",450 "ttft_p50_ms",451 "ttft_p95_ms",452 "latency_p95_ms",453 "error_rate",454 )455 }456 for r in concurrency457 ],458 "peak_memory_bytes": (end_stats or {}).get("mlx", {}).get("peak_bytes"),459 "recommended": {"max_active": best["clients"] if best else None},460 },461 "single": {"ttft_ms": single_s["ttft_ms_p50"], "tokens_per_second": single_s["tokens_per_second_p50"]},462 "scenarios": scenarios,463 "prefix_cache": prefix_cache,464 "concurrency": concurrency,465 "peak_memory_bytes": (end_stats or {}).get("mlx", {}).get("peak_bytes"),466 "engine_start": start_stats,467 "engine_end": end_stats,468 "memory_end": memory_state(),469 "samples_single": [asdict(s) | {"text": None} for s in single],470 }471 out_dir = Path(args.out).expanduser()472 out_dir.mkdir(parents=True, exist_ok=True)473 path = out_dir / f"{time.strftime('%Y%m%d-%H%M%S')}-{args.label}.json"474 path.write_text(json.dumps(report, ensure_ascii=False, indent=1))475 print(f"rapport : {path}")476 return report477478479def main() -> None:480 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)481 ap.add_argument("--base", default="http://127.0.0.1:8811")482 ap.add_argument("--stats", default=None, help="URL /internal/stats du moteur (mémoire MLX)")483 ap.add_argument("--model", default="default_model")484 ap.add_argument("--key", default=None)485 ap.add_argument("--label", required=True)486 ap.add_argument("--out", default="~/maclustr-llm-data/bench")487 ap.add_argument("--n", type=int, default=5)488 ap.add_argument("--concurrency", default="1,2,4,8,16")489 ap.add_argument("--concurrency-tokens", type=int, default=256)490 ap.add_argument("--rounds", type=int, default=2)491 ap.add_argument(492 "--quick", action="store_true", help="saute les scénarios longs (génération longue, grands contextes)"493 )494 ap.add_argument(495 "--extra",496 default=None,497 help='JSON fusionné dans chaque requête, ex. \'{"chat_template_kwargs":{"enable_thinking":false}}\'',498 )499 asyncio.run(run(ap.parse_args()))500501502if __name__ == "__main__":503 main()504