SPB Git

Repository Public

bench.py 19.6 KB 504 lines python

1"""Banc d'essai reproductible de MacLustr LLM API.23Mesure, contre un moteur MLX (ou la passerelle), les scénarios du cahier des charges : prompt court, prompt4long, conversation multi-tours, génération longue, programmation, raisonnement, tool calling, préfixes répétés,5contexte volumineux et montée en concurrence (1, 2, 4, 8, 16 clients).67    python bench/bench.py --base http://127.0.0.1:8811 --stats http://127.0.0.1:8811/internal/stats \8        --label instruct-2507-4bit --out ~/maclustr-llm-data/bench910Chaque valeur du rapport est mesurée pendant l'exécution ; rien n'est estimé. Les temps sont pris côté client :11TTFT = délai entre l'envoi et le premier delta de contenu ; tokens/s = (tokens − 1) / (dernier − premier delta).12"""1314from __future__ import annotations1516import argparse17import asyncio18import json19import platform20import statistics21import subprocess22import time23from dataclasses import asdict, dataclass, field24from pathlib import Path25from typing import Any2627import httpx2829FILLER = (30    "Le calcul distribué sur Apple Silicon repose sur la mémoire unifiée : le processeur central et le "31    "processeur graphique partagent les mêmes pages, ce qui évite les copies entre mémoire système et mémoire "32    "vidéo. Un modèle de langage quantifié sur quatre bits y tient donc en entier, et le débit dépend surtout "33    "de la bande passante mémoire. "34)353637def long_text(approx_tokens: int, salt: str = "") -> str:38    """Texte français d'environ `approx_tokens` tokens (≈ 120 tokens mesurés par phrase de remplissage)."""39    n = max(1, approx_tokens // 120)40    return " ".join(f"[{salt}{i}] {FILLER}" for i in range(n))414243@dataclass44class Sample:45    ok: bool46    ttft_ms: float | None = None47    latency_ms: float = 0.048    gen_ms: float = 0.049    prompt_tokens: int = 050    completion_tokens: int = 051    cached_tokens: int = 052    finish_reason: str | None = None53    tool_calls: int = 054    error: str | None = None55    text: str = field(default="", repr=False)5657    @property58    def tps(self) -> float | None:59        if self.completion_tokens > 1 and self.gen_ms > 0:60            return (self.completion_tokens - 1) / (self.gen_ms / 1000)61        return None626364class Client:65    def __init__(self, base: str, model: str, key: str | None) -> None:66        self.base = base.rstrip("/")67        self.model = model68        headers = {"Authorization": f"Bearer {key}"} if key else {}69        self.http = httpx.AsyncClient(70            timeout=httpx.Timeout(900, connect=5), headers=headers, limits=httpx.Limits(max_connections=64)71        )7273    async def chat(self, messages: list[dict[str, Any]], max_tokens: int = 256, **extra: Any) -> Sample:74        payload = {75            "model": self.model,76            "messages": messages,77            "max_tokens": max_tokens,78            "stream": True,79            "stream_options": {"include_usage": True},80            "temperature": 0.0,81            **extra,82        }83        s = Sample(ok=False)84        t0 = time.perf_counter()85        first = last = None86        try:87            async with self.http.stream("POST", f"{self.base}/v1/chat/completions", json=payload) as r:88                if r.status_code != 200:89                    s.error = f"HTTP {r.status_code}: {(await r.aread())[:200]!r}"90                    return s91                async for line in r.aiter_lines():92                    if not line.startswith("data:"):93                        continue94                    data = line[5:].strip()95                    if data == "[DONE]":96                        break97                    evt = json.loads(data)98                    if "error" in evt:99                        s.error = str(evt["error"])[:200]100                        return s101                    if evt.get("usage"):102                        u = evt["usage"]103                        s.prompt_tokens, s.completion_tokens = u["prompt_tokens"], u["completion_tokens"]104                        s.cached_tokens = (u.get("prompt_tokens_details") or {}).get("cached_tokens", 0)105                    for ch in evt.get("choices") or []:106                        d = ch.get("delta") or {}107                        piece = d.get("content") or d.get("reasoning") or d.get("reasoning_content")108                        if piece or d.get("tool_calls"):109                            now = time.perf_counter()110                            first = first or now111                            last = now112                            s.text += d.get("content") or ""113                            s.tool_calls += len(d.get("tool_calls") or [])114                        if ch.get("finish_reason"):115                            s.finish_reason = ch["finish_reason"]116            s.ok = True117        except Exception as exc:  # noqa: BLE001 - toute panne compte comme une erreur mesurée118            s.error = f"{type(exc).__name__}: {exc}"[:200]119        s.latency_ms = (time.perf_counter() - t0) * 1000120        if first is not None and last is not None:121            s.ttft_ms = (first - t0) * 1000122            s.gen_ms = (last - first) * 1000123        return s124125126def pct(vals: list[float], p: float) -> float | None:127    if not vals:128        return None129    vals = sorted(vals)130    return round(vals[min(len(vals) - 1, int(round(p * (len(vals) - 1))))], 1)131132133def summarize(name: str, description: str, samples: list[Sample], wall_s: float | None = None) -> dict[str, Any]:134    ok = [s for s in samples if s.ok]135    ttft = [s.ttft_ms for s in ok if s.ttft_ms is not None]136    lat = [s.latency_ms for s in ok]137    tps = [t for s in ok if (t := s.tps) is not None]138    out: dict[str, Any] = {139        "name": name,140        "description": description,141        "requests": len(samples),142        "error_rate": round(1 - len(ok) / len(samples), 4) if samples else None,143        "ttft_ms_p50": pct(ttft, 0.5),144        "ttft_ms_p95": pct(ttft, 0.95),145        "ttft_ms_p99": pct(ttft, 0.99),146        "latency_ms_p50": pct(lat, 0.5),147        "latency_ms_p95": pct(lat, 0.95),148        "latency_ms_p99": pct(lat, 0.99),149        "tokens_per_second_p50": pct(tps, 0.5),150        "prompt_tokens_avg": round(statistics.mean(s.prompt_tokens for s in ok)) if ok else None,151        "completion_tokens_avg": round(statistics.mean(s.completion_tokens for s in ok)) if ok else None,152        "cached_tokens_avg": round(statistics.mean(s.cached_tokens for s in ok)) if ok else None,153        "cache_hit_ratio": round(sum(s.cached_tokens for s in ok) / max(1, sum(s.prompt_tokens for s in ok)), 3)154        if ok155        else None,156        "errors": sorted({s.error for s in samples if s.error})[:3],157    }158    if wall_s:159        out["wall_s"] = round(wall_s, 2)160        out["aggregate_tps"] = round(sum(s.completion_tokens for s in ok) / wall_s, 1)161    return out162163164def sh(cmd: list[str]) -> str:165    try:166        return subprocess.run(cmd, capture_output=True, text=True, timeout=5, check=False).stdout.strip()167    except (OSError, subprocess.SubprocessError):168        return ""169170171def memory_state() -> dict[str, Any]:172    level = sh(["/usr/sbin/sysctl", "-n", "kern.memorystatus_vm_pressure_level"])173    free = sh(["/usr/sbin/sysctl", "-n", "kern.memorystatus_level"])174    swap = sh(["/usr/sbin/sysctl", "-n", "vm.swapusage"])175    return {176        "pressure_level": {"1": "normal", "2": "warn", "4": "critical"}.get(level, level),177        "free_percent": int(free) if free.isdigit() else None,178        "swap": swap,179    }180181182WEATHER_TOOL = [183    {184        "type": "function",185        "function": {186            "name": "get_weather",187            "description": "Retourne la météo actuelle d'une ville.",188            "parameters": {189                "type": "object",190                "properties": {191                    "city": {"type": "string"},192                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},193                },194                "required": ["city"],195            },196        },197    }198]199200201async def run(args: argparse.Namespace) -> dict[str, Any]:202    c = Client(args.base, args.model, args.key)203    stats_http = httpx.AsyncClient(timeout=5)204205    async def engine_stats() -> dict[str, Any] | None:206        if not args.stats:207            return None208        try:209            return (await stats_http.get(args.stats)).json()210        except Exception:  # noqa: BLE001211            return None212213    scenarios: list[dict[str, Any]] = []214    extra = json.loads(args.extra) if args.extra else {}215216    async def seq(name: str, desc: str, build: Any, n: int, max_tokens: int, **kw: Any) -> list[Sample]:217        samples = [await c.chat(build(i), max_tokens=max_tokens, **extra, **kw) for i in range(n)]218        scenarios.append(summarize(name, desc, samples))219        print(json.dumps(scenarios[-1], ensure_ascii=False), flush=True)220        return samples221222    # réchauffage (hors mesure)223    await c.chat([{"role": "user", "content": "Bonjour."}], max_tokens=16, **extra)224    start_stats = await engine_stats()225226    single = await seq(227        "short_prompt",228        "Question courte, réponse de 128 tokens.",229        lambda i: [230            {231                "role": "user",232                "content": f"Explique en quelques phrases le théorème n° {i + 1} de ton choix en mathématiques.",233            }234        ],235        args.n,236        128,237    )238    if not args.quick:239        await seq(240            "long_generation",241            "Génération longue de 1 024 tokens.",242            lambda i: [243                {244                    "role": "user",245                    "content": f"Rédige un essai détaillé (variante {i}) sur l'histoire de "246                    "l'informatique, des origines à aujourd'hui.",247                }248            ],249            3,250            1024,251        )252        await seq(253            "long_prompt",254            "Prompt d'environ 4 000 tokens (jamais vu), réponse de 128 tokens.",255            lambda i: [{"role": "user", "content": long_text(4000, f"L{i}-") + "\n\nRésume ce texte en 3 phrases."}],256            3,257            128,258        )259        await seq(260            "large_context",261            "Contexte d'environ 16 000 tokens (jamais vu), réponse de 128 tokens.",262            lambda i: [{"role": "user", "content": long_text(16000, f"X{i}-") + "\n\nRésume ce texte en 3 phrases."}],263            2,264            128,265        )266    await seq(267        "coding",268        "Programmation : fonction Python avec tests.",269        lambda i: [270            {271                "role": "user",272                "content": "Écris une fonction Python `merge_intervals(intervals)` qui "273                "fusionne des intervalles qui se chevauchent, avec docstring "274                f"et 3 tests. Variante {i}.",275            }276        ],277        3,278        512,279    )280    await seq(281        "reasoning",282        "Raisonnement en plusieurs étapes.",283        lambda i: [284            {285                "role": "user",286                "content": f"Un train part à {8 + i} h à 90 km/h, un second part une heure "287                "plus tard du même point à 120 km/h. À quelle heure le second "288                "rattrape-t-il le premier ? Détaille le raisonnement.",289            }290        ],291        3,292        400,293    )294    tool_samples = await seq(295        "tool_calling",296        "Appel d'outil (get_weather) attendu.",297        lambda i: [298            {299                "role": "user",300                "content": [301                    "Quel temps fait-il à Québec ?",302                    "Donne-moi la météo de Montréal en celsius.",303                    "Météo à Paris ?",304                ][i % 3],305            }306        ],307        3,308        200,309        tools=WEATHER_TOOL,310    )311    scenarios[-1]["tool_call_rate"] = round(sum(1 for s in tool_samples if s.tool_calls > 0) / len(tool_samples), 2)312313    # conversation multi-tours : chaque tour réutilise le préfixe du précédent (cache KV)314    history: list[dict[str, Any]] = [{"role": "system", "content": "Tu es un assistant concis."}]315    turns: list[Sample] = []316    for q in [317        "Présente Montréal en 4 phrases.",318        "Et son climat ?",319        "Quels quartiers visiter ?",320        "Compare avec Québec.",321        "Résume notre échange en 3 points.",322    ]:323        history.append({"role": "user", "content": q})324        s = await c.chat(history, max_tokens=160, **extra)325        turns.append(s)326        history.append({"role": "assistant", "content": s.text})327    scenarios.append(summarize("multi_turn", "Conversation de 5 tours (préfixe réutilisé d'un tour à l'autre).", turns))328    print(json.dumps(scenarios[-1], ensure_ascii=False), flush=True)329330    # préfixes répétés : même prompt système d'environ 3 000 tokens, questions différentes331    system = "Tu es l'assistant d'une documentation technique. Contexte :\n" + long_text(3000, "DOC-")332    cold = await c.chat(333        [334            {"role": "system", "content": system},335            {"role": "user", "content": "Question 0 : de quoi parle ce contexte ?"},336        ],337        max_tokens=64,338        **extra,339    )340    warm = [341        await c.chat(342            [343                {"role": "system", "content": system},344                {"role": "user", "content": f"Question {i} : cite un mot-clé du contexte."},345            ],346            max_tokens=64,347            **extra,348        )349        for i in range(1, 6)350    ]351    scenarios.append(352        summarize(353            "repeated_prefix",354            "Préfixe système commun d'environ 3 000 tokens, 5 questions différentes après un premier appel à froid.",355            warm,356        )357    )358    prefix_cache = {359        "cold_ttft_ms": round(cold.ttft_ms, 1) if cold.ttft_ms else None,360        "warm_ttft_ms": pct([s.ttft_ms for s in warm if s.ttft_ms], 0.5),361        "warm_cached_tokens": warm[-1].cached_tokens,362        "prompt_tokens": warm[-1].prompt_tokens,363    }364    print(json.dumps({**scenarios[-1], "prefix_cache": prefix_cache}, ensure_ascii=False), flush=True)365366    # montée en concurrence367    concurrency: list[dict[str, Any]] = []368    for n in [int(x) for x in args.concurrency.split(",")]:369        before = await engine_stats()370        mem_before = memory_state()371        t0 = time.perf_counter()372        samples: list[Sample] = []373        for r in range(args.rounds):  # chaque manche lance exactement n générations simultanées374            samples += list(375                await asyncio.gather(376                    *[377                        c.chat(378                            [379                                {380                                    "role": "user",381                                    "content": f"Rédige un texte informatif (sujet n° {r * 100 + i}) "382                                    "d'environ 300 mots sur une invention scientifique de ton choix.",383                                }384                            ],385                            max_tokens=args.concurrency_tokens,386                            **extra,387                        )388                        for i in range(n)389                    ]390                )391            )392        wall = time.perf_counter() - t0393        after = await engine_stats()394        s = summarize(395            f"concurrency_{n}", f"{n} générations simultanées de {args.concurrency_tokens} tokens.", samples, wall396        )397        row = {398            "clients": n,399            "aggregate_tps": s["aggregate_tps"],400            "per_request_tps": s["tokens_per_second_p50"],401            "ttft_p50_ms": s["ttft_ms_p50"],402            "ttft_p95_ms": s["ttft_ms_p95"],403            "latency_p50_ms": s["latency_ms_p50"],404            "latency_p95_ms": s["latency_ms_p95"],405            "latency_p99_ms": s["latency_ms_p99"],406            "error_rate": s["error_rate"],407            "wall_s": s["wall_s"],408            "mlx_peak_bytes": (after or {}).get("mlx", {}).get("peak_bytes"),409            "mlx_active_bytes": (after or {}).get("mlx", {}).get("active_bytes"),410            "memory_before": mem_before,411            "memory_after": memory_state(),412            "errors": s["errors"],413        }414        del before415        concurrency.append(row)416        print(json.dumps(row, ensure_ascii=False), flush=True)417418    end_stats = await engine_stats()419    ok_rows = [r for r in concurrency if r["error_rate"] == 0 and r["memory_after"]["pressure_level"] == "normal"]420    best = max(ok_rows, key=lambda r: r["aggregate_tps"]) if ok_rows else None421    single_s = summarize("single", "", single)422    chip = sh(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"])423    mem = sh(["/usr/sbin/sysctl", "-n", "hw.memsize"])424    report = {425        "meta": {426            "date": time.strftime("%Y-%m-%d %H:%M:%S"),427            "label": args.label,428            "host": platform.node(),429            "chip": chip,430            "memory_gb": round(int(mem) / (1 << 30)) if mem.isdigit() else None,431            "macos": platform.mac_ver()[0],432            "mlx": (end_stats or {}).get("versions", {}).get("mlx"),433            "mlx_lm": (end_stats or {}).get("versions", {}).get("mlx_lm"),434            "model": (end_stats or {}).get("model"),435            "base": args.base,436            "extra": extra,437            "decode_concurrency": (end_stats or {}).get("decode_concurrency"),438            "prompt_concurrency": (end_stats or {}).get("prompt_concurrency"),439        },440        "summary": {441            "model": (end_stats or {}).get("model") or args.label,442            "single": {"ttft_ms": single_s["ttft_ms_p50"], "tokens_per_second": single_s["tokens_per_second_p50"]},443            "concurrency": [444                {445                    k: r[k]446                    for k in (447                        "clients",448                        "aggregate_tps",449                        "per_request_tps",450                        "ttft_p50_ms",451                        "ttft_p95_ms",452                        "latency_p95_ms",453                        "error_rate",454                    )455                }456                for r in concurrency457            ],458            "peak_memory_bytes": (end_stats or {}).get("mlx", {}).get("peak_bytes"),459            "recommended": {"max_active": best["clients"] if best else None},460        },461        "single": {"ttft_ms": single_s["ttft_ms_p50"], "tokens_per_second": single_s["tokens_per_second_p50"]},462        "scenarios": scenarios,463        "prefix_cache": prefix_cache,464        "concurrency": concurrency,465        "peak_memory_bytes": (end_stats or {}).get("mlx", {}).get("peak_bytes"),466        "engine_start": start_stats,467        "engine_end": end_stats,468        "memory_end": memory_state(),469        "samples_single": [asdict(s) | {"text": None} for s in single],470    }471    out_dir = Path(args.out).expanduser()472    out_dir.mkdir(parents=True, exist_ok=True)473    path = out_dir / f"{time.strftime('%Y%m%d-%H%M%S')}-{args.label}.json"474    path.write_text(json.dumps(report, ensure_ascii=False, indent=1))475    print(f"rapport : {path}")476    return report477478479def main() -> None:480    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)481    ap.add_argument("--base", default="http://127.0.0.1:8811")482    ap.add_argument("--stats", default=None, help="URL /internal/stats du moteur (mémoire MLX)")483    ap.add_argument("--model", default="default_model")484    ap.add_argument("--key", default=None)485    ap.add_argument("--label", required=True)486    ap.add_argument("--out", default="~/maclustr-llm-data/bench")487    ap.add_argument("--n", type=int, default=5)488    ap.add_argument("--concurrency", default="1,2,4,8,16")489    ap.add_argument("--concurrency-tokens", type=int, default=256)490    ap.add_argument("--rounds", type=int, default=2)491    ap.add_argument(492        "--quick", action="store_true", help="saute les scénarios longs (génération longue, grands contextes)"493    )494    ap.add_argument(495        "--extra",496        default=None,497        help='JSON fusionné dans chaque requête, ex. \'{"chat_template_kwargs":{"enable_thinking":false}}\'',498    )499    asyncio.run(run(ap.parse_args()))500501502if __name__ == "__main__":503    main()504
Files main
Menu

Theme