Repository Public
1"""Évaluation de qualité à correction automatique, pour comparer des modèles candidats sur le même matériel.23Six catégories (français, anglais, raisonnement, programmation, tool calling, respect des instructions),4chaque item a un vérificateur déterministe : nombre attendu, mots-clés, exécution de tests sur le code produit,5appel d'outil attendu, contrainte de format. Température 0. Ce n'est pas un benchmark académique : c'est un6filtre reproductible pour choisir entre des candidats, complété par les mesures de vitesse de bench.py.78 python bench/quality.py --base http://127.0.0.1:8811 --label instruct-2507-4bit9"""1011from __future__ import annotations1213import argparse14import asyncio15import json16import re17import subprocess18import sys19import tempfile20import time21import unicodedata22from collections.abc import Callable23from pathlib import Path24from typing import Any2526import httpx2728Check = Callable[[dict[str, Any]], bool]293031def norm(text: str) -> str:32 text = unicodedata.normalize("NFKD", text.lower())33 return "".join(c for c in text if not unicodedata.combining(c))343536def has(*words: str) -> Check:37 return lambda r: all(norm(w) in norm(r["content"]) for w in words)383940def has_any(*words: str) -> Check:41 return lambda r: any(norm(w) in norm(r["content"]) for w in words)424344def number(expected: float, tol: float = 1e-6) -> Check:45 def check(r: dict[str, Any]) -> bool:46 m = re.findall(r"(?:reponse|answer)\s*:?\s*\**\s*(-?\d[\d\s.,]*)", norm(r["content"]))47 if not m:48 return False49 raw = m[-1].strip().rstrip(".,").replace(" ", "").replace(" ", "")50 if "," in raw and "." not in raw:51 raw = raw.replace(",", ".")52 raw = raw.replace(",", "")53 try:54 return abs(float(raw) - expected) <= tol55 except ValueError:56 return False5758 return check596061def code(func: str, tests: str) -> Check:62 def check(r: dict[str, Any]) -> bool:63 blocks = re.findall(r"```(?:python|py)?\s*\n(.*?)```", r["content"], re.S)64 src = max(blocks, key=len) if blocks else r["content"]65 if func not in src:66 return False67 with tempfile.TemporaryDirectory() as d:68 path = Path(d) / "sol.py"69 path.write_text(src + "\n\n" + tests + "\nprint('TESTS_OK')\n")70 try:71 out = subprocess.run(72 [sys.executable, "-I", str(path)], capture_output=True, text=True, timeout=10, cwd=d, check=False73 )74 except subprocess.TimeoutExpired:75 return False76 return "TESTS_OK" in out.stdout7778 return check798081def tool(name: str | None, **args: Any) -> Check:82 def check(r: dict[str, Any]) -> bool:83 calls = r["tool_calls"]84 if name is None:85 return not calls86 for c in calls:87 if c["name"] != name:88 continue89 try:90 got = json.loads(c["arguments"]) if isinstance(c["arguments"], str) else c["arguments"]91 except ValueError:92 return False93 if all(94 norm(str(got.get(k, ""))) == norm(str(v)) or norm(str(v)) in norm(str(got.get(k, "")))95 for k, v in args.items()96 ):97 return True98 return False99100 return check101102103def fmt(pred: Callable[[str], bool]) -> Check:104 return lambda r: pred(r["content"].strip())105106107def is_json_with(*keys: str) -> Callable[[str], bool]:108 def pred(text: str) -> bool:109 text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())110 try:111 obj = json.loads(text)112 except ValueError:113 return False114 return isinstance(obj, dict) and set(obj) == set(keys)115116 return pred117118119TOOLS = [120 {121 "type": "function",122 "function": {123 "name": "get_weather",124 "description": "Météo actuelle d'une ville.",125 "parameters": {126 "type": "object",127 "properties": {128 "city": {"type": "string"},129 "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},130 },131 "required": ["city"],132 },133 },134 },135 {136 "type": "function",137 "function": {138 "name": "search_flights",139 "description": "Recherche des vols entre deux villes.",140 "parameters": {141 "type": "object",142 "properties": {143 "origin": {"type": "string"},144 "destination": {"type": "string"},145 "date": {"type": "string", "description": "AAAA-MM-JJ"},146 },147 "required": ["origin", "destination", "date"],148 },149 },150 },151 {152 "type": "function",153 "function": {154 "name": "convert_currency",155 "description": "Convertit un montant.",156 "parameters": {157 "type": "object",158 "properties": {"amount": {"type": "number"}, "from": {"type": "string"}, "to": {"type": "string"}},159 "required": ["amount", "from", "to"],160 },161 },162 },163]164NUM = " Termine par une ligne « Réponse : <nombre> »."165NUM_EN = ' End with a line "Answer: <number>".'166167ITEMS: list[tuple[str, str, Check, dict[str, Any]]] = [168 # --- français169 ("fr", "Quelle est la capitale du Canada ? Réponds en une phrase.", has("ottawa"), {}),170 (171 "fr",172 "Conjugue le verbe « aller » au subjonctif présent, à la première personne du pluriel. Donne seulement "173 "la forme conjuguée avec son pronom.",174 has("que nous allions"),175 {},176 ),177 (178 "fr",179 "Donne le pluriel de « un cheval », « un bail » et « un festival », séparés par des virgules.",180 has("chevaux", "baux", "festivals"),181 {},182 ),183 (184 "fr",185 'Traduis en français : "The meeting was postponed until next Thursday because of the snowstorm."',186 lambda r: (187 has_any("reportee", "remise", "repoussee", "ajournee")(r)188 and has("jeudi prochain")(r)189 and has_any("tempete de neige")(r)190 ),191 {},192 ),193 (194 "fr",195 "Dans la phrase « Les pommes que j'ai mangées étaient mûres », pourquoi « mangées » s'accorde-t-il ? "196 "Réponds en deux phrases.",197 has_any("cod", "complement d'objet direct", "complement direct"),198 {},199 ),200 (201 "fr",202 "Résume en une seule phrase : « L'entreprise a annoncé mardi la fermeture de son usine de Trois-Rivières, "203 "ce qui entraînera la perte de 320 emplois d'ici la fin de l'année, en raison de la baisse de la demande. »",204 lambda r: has("320")(r) and has_any("trois-rivieres")(r) and len(r["content"]) < 400,205 {},206 ),207 # --- anglais208 ("en", "What is the past participle of 'to write'? Answer in one short sentence.", has("written"), {}),209 (210 "en",211 "Translate into English: « Je n'ai pas eu le temps de finir le rapport avant la réunion. »",212 lambda r: (213 has("report")(r)214 and has("meeting")(r)215 and has_any("didn't have time", "did not have time", "didn't have the time", "did not have the time")(r)216 ),217 {},218 ),219 (220 "en",221 "Which word is the odd one out: apple, banana, carrot, cherry? Answer with the word and a short reason.",222 has("carrot"),223 {},224 ),225 (226 "en",227 'Rewrite in the passive voice: "The committee approved the budget."',228 has("the budget was approved by the committee"),229 {},230 ),231 (232 "en",233 'Summarize in one sentence: "Researchers at the university discovered that the new alloy is 40% lighter '234 'than steel while keeping the same strength, which could cut fuel use in aircraft."',235 lambda r: has("40")(r) and has_any("alloy")(r) and len(r["content"]) < 400,236 {},237 ),238 # --- raisonnement239 (240 "reasoning",241 "Un magasin vend un article 80 $. Il augmente le prix de 25 %, puis accorde un rabais de 20 % sur "242 "le nouveau prix. Quel est le prix final en dollars ?" + NUM,243 number(80),244 {},245 ),246 (247 "reasoning",248 "Trois ouvriers construisent un mur en 12 jours. Combien de jours faut-il à 4 ouvriers travaillant "249 "au même rythme ?" + NUM,250 number(9),251 {},252 ),253 (254 "reasoning",255 "Marie a deux fois l'âge de Paul. Dans 10 ans, la somme de leurs âges sera 50. Quel âge a Marie "256 "aujourd'hui ?" + NUM,257 number(20),258 {},259 ),260 ("reasoning", "Combien y a-t-il de nombres premiers entre 20 et 50 ?" + NUM, number(7), {}),261 (262 "reasoning",263 "Une horloge sonne 6 coups en 5 secondes (intervalles réguliers). Combien de secondes lui "264 "faut-il pour sonner 12 coups ?" + NUM,265 number(11),266 {},267 ),268 (269 "reasoning",270 "A train travels 150 km at 60 km/h, then 100 km at 50 km/h. What is the average speed for the "271 "whole trip in km/h, rounded to one decimal?" + NUM_EN,272 number(55.6, 0.06),273 {},274 ),275 (276 "reasoning",277 "If all bloops are razzies and all razzies are lazzies, are all bloops definitely lazzies? "278 "Answer yes or no, then one sentence.",279 lambda r: norm(r["content"]).lstrip(" *").startswith("yes"),280 {},281 ),282 (283 "reasoning",284 "Dans une course, tu dépasses le coureur qui est en deuxième position. En quelle position es-tu "285 "maintenant ? Réponds par un chiffre." + NUM,286 number(2),287 {},288 ),289 (290 "reasoning",291 "J'ai 3 pommes. J'en mange une, j'en achète 5, puis j'en donne la moitié de ce que j'ai. "292 "Combien m'en reste-t-il ?" + NUM,293 number(3.5, 0.51),294 {},295 ),296 # --- programmation297 (298 "code",299 "Écris une fonction Python `fizzbuzz(n)` qui retourne la liste des chaînes de 1 à n : « Fizz » pour les "300 "multiples de 3, « Buzz » pour ceux de 5, « FizzBuzz » pour les deux, sinon le nombre en chaîne. "301 "Donne uniquement le code dans un bloc ```python.",302 code("def fizzbuzz", "assert fizzbuzz(15)[-1]=='FizzBuzz' and fizzbuzz(5)==['1','2','Fizz','4','Buzz']"),303 {},304 ),305 (306 "code",307 "Write a Python function `merge_intervals(intervals)` that merges overlapping intervals given as a list "308 "of [start, end] lists and returns them sorted. Only the code in a ```python block.",309 code(310 "def merge_intervals",311 "assert [list(x) for x in merge_intervals([[1,3],[2,6],[8,10],[15,18]])]==[[1,6],[8,10],[15,18]]\n"312 "assert [list(x) for x in merge_intervals([[1,4],[4,5]])]==[[1,5]]\nassert merge_intervals([])==[]",313 ),314 {},315 ),316 (317 "code",318 "Write a Python function `is_valid_parentheses(s)` returning True if the string of ()[]{} is balanced. "319 "Only the code in a ```python block.",320 code(321 "def is_valid_parentheses",322 "assert is_valid_parentheses('()[]{}') and is_valid_parentheses('{[]}') and "323 "not is_valid_parentheses('(]') and not is_valid_parentheses('((') and is_valid_parentheses('')",324 ),325 {},326 ),327 (328 "code",329 "Écris une fonction Python `lru(capacity)` qui retourne un objet avec les méthodes `get(key)` (retourne "330 "-1 si absent) et `put(key, value)`, implémentant un cache LRU. Uniquement le code dans un bloc ```python.",331 code(332 "def lru",333 "c=lru(2); c.put(1,1); c.put(2,2); assert c.get(1)==1; c.put(3,3); assert c.get(2)==-1\n"334 "c.put(4,4); assert c.get(1)==-1 and c.get(3)==3 and c.get(4)==4",335 ),336 {},337 ),338 (339 "code",340 "Write a Python function `roman_to_int(s)` converting a Roman numeral to an integer. Only the code in a "341 "```python block.",342 code(343 "def roman_to_int",344 "assert roman_to_int('MCMXCIV')==1994 and roman_to_int('LVIII')==58 and roman_to_int('IX')==9",345 ),346 {},347 ),348 (349 "code",350 "This Python function has a bug. Fix it and return only the corrected code in a ```python block.\n"351 "```python\ndef median(xs):\n xs.sort()\n n = len(xs)\n if n % 2 == 0:\n return xs[n // 2]\n"352 " return (xs[n // 2 - 1] + xs[n // 2]) / 2\n```",353 code("def median", "assert median([3,1,2])==2 and median([4,1,3,2])==2.5 and median([5])==5"),354 {},355 ),356 (357 "code",358 "Write a Python function `top_k_words(text, k)` that returns the k most frequent lowercase words as a "359 "list of (word, count) tuples, ties broken alphabetically. Words are sequences of letters. Only the code "360 "in a ```python block.",361 code(362 "def top_k_words",363 "assert [tuple(x) for x in top_k_words('a b b c c c A', 2)]==[('c',3),('a',2)]\n"364 "assert [tuple(x) for x in top_k_words('x y', 5)]==[('x',1),('y',1)]",365 ),366 {},367 ),368 # --- tool calling369 ("tools", "Quel temps fait-il à Québec en ce moment ?", tool("get_weather", city="Québec"), {"tools": TOOLS}),370 (371 "tools",372 "Donne-moi la météo de Tokyo en fahrenheit.",373 tool("get_weather", city="Tokyo", unit="fahrenheit"),374 {"tools": TOOLS},375 ),376 (377 "tools",378 "Trouve-moi un vol de Montréal à Paris le 14 février 2027.",379 tool("search_flights", origin="Montréal", destination="Paris", date="2027-02-14"),380 {"tools": TOOLS},381 ),382 (383 "tools",384 "Combien font 250 dollars canadiens en euros ?",385 tool("convert_currency", amount=250, to="EUR"),386 {"tools": TOOLS},387 ),388 ("tools", "Explique-moi en une phrase ce qu'est une API REST.", tool(None), {"tools": TOOLS}),389 ("tools", "Merci, c'est tout pour aujourd'hui !", tool(None), {"tools": TOOLS}),390 (391 "tools",392 "I'm flying from Toronto to Vancouver on 2027-03-02, can you look for flights?",393 tool("search_flights", origin="Toronto", destination="Vancouver", date="2027-03-02"),394 {"tools": TOOLS},395 ),396 # --- respect des instructions397 (398 "instructions",399 "Réponds par un seul mot, sans ponctuation : quelle est la couleur du ciel par temps clair ?",400 fmt(lambda t: norm(t) in ("bleu", "bleue")),401 {},402 ),403 (404 "instructions",405 'Retourne uniquement un objet JSON avec exactement les clés "nom" et "age" pour une personne '406 "fictive. Aucun autre texte.",407 fmt(is_json_with("nom", "age")),408 {},409 ),410 (411 "instructions",412 "Liste exactement trois fruits, un par ligne, sans numérotation, sans puce et sans autre texte.",413 fmt(414 lambda t: (415 len([x for x in t.splitlines() if x.strip()]) == 3 and not re.search(r"^\s*(?:[-*•]|\d+[.)])", t, re.M)416 )417 ),418 {},419 ),420 (421 "instructions",422 "ANSWER ONLY IN UPPERCASE: what is the largest planet of the solar system?",423 fmt(lambda t: "JUPITER" in t and t == t.upper()),424 {},425 ),426 (427 "instructions",428 "Réponds en anglais uniquement, même si la question est en français : quel est le plus grand océan du monde ?",429 lambda r: has("pacific")(r) and not has_any("pacifique", "l'ocean")(r),430 {},431 ),432 (433 "instructions",434 "Écris une phrase de exactement 7 mots sur l'hiver. Rien d'autre.",435 fmt(lambda t: len(re.findall(r"[\w'’-]+", t.strip().strip('."«» '))) == 7),436 {},437 ),438 (439 "instructions",440 "Tu es un assistant qui ne parle que de cuisine. Si on te demande autre chose, réponds exactement "441 "« Je ne parle que de cuisine. »\n\nQuestion : qui a gagné la Coupe du monde 2018 ?",442 fmt(lambda t: norm(t).strip('"« »').startswith("je ne parle que de cuisine")),443 {},444 ),445]446447448async def ask(449 http: httpx.AsyncClient, base: str, model: str, prompt: str, extra: dict[str, Any], max_tokens: int450) -> dict[str, Any]:451 payload = {452 "model": model,453 "messages": [{"role": "user", "content": prompt}],454 "max_tokens": max_tokens,455 "temperature": 0.0,456 **extra,457 }458 t0 = time.perf_counter()459 r = await http.post(f"{base}/v1/chat/completions", json=payload)460 r.raise_for_status()461 body = r.json()462 msg = body["choices"][0]["message"]463 calls = [464 {"name": c["function"]["name"], "arguments": c["function"]["arguments"]} for c in msg.get("tool_calls") or []465 ]466 return {467 "content": msg.get("content") or "",468 "tool_calls": calls,469 "usage": body.get("usage", {}),470 "seconds": time.perf_counter() - t0,471 "finish": body["choices"][0].get("finish_reason"),472 }473474475async def main_async(args: argparse.Namespace) -> None:476 extra = json.loads(args.extra) if args.extra else {}477 headers = {"Authorization": f"Bearer {args.key}"} if args.key else {}478 results: list[dict[str, Any]] = []479 async with httpx.AsyncClient(timeout=600, headers=headers) as http:480 sem = asyncio.Semaphore(args.parallel)481482 async def one(i: int, cat: str, prompt: str, check: Check, kw: dict[str, Any]) -> None:483 async with sem:484 try:485 r = await ask(http, args.base, args.model, prompt, {**extra, **kw}, args.max_tokens)486 ok = bool(check(r))487 except Exception as exc: # noqa: BLE001 - une panne compte comme un échec mesuré488 r, ok = (489 {490 "content": f"ERREUR {type(exc).__name__}: {exc}",491 "tool_calls": [],492 "usage": {},493 "seconds": 0,494 "finish": None,495 },496 False,497 )498 results.append(499 {500 "i": i,501 "category": cat,502 "ok": ok,503 "prompt": prompt,504 "answer": r["content"][:1500],505 "tool_calls": r["tool_calls"],506 "seconds": round(r["seconds"], 2),507 "completion_tokens": r["usage"].get("completion_tokens"),508 "finish": r["finish"],509 }510 )511512 await asyncio.gather(*(one(i, *item) for i, item in enumerate(ITEMS)))513 results.sort(key=lambda r: r["i"])514 cats: dict[str, list[bool]] = {}515 for r in results:516 cats.setdefault(r["category"], []).append(r["ok"])517 scores = {c: {"passed": sum(v), "total": len(v), "score": round(sum(v) / len(v), 3)} for c, v in cats.items()}518 total = {"passed": sum(r["ok"] for r in results), "total": len(results)}519 total["score"] = round(total["passed"] / total["total"], 3)520 report = {521 "meta": {522 "date": time.strftime("%Y-%m-%d %H:%M:%S"),523 "label": args.label,524 "extra": extra,525 "items": len(ITEMS),526 "temperature": 0.0,527 "max_tokens": args.max_tokens,528 },529 "scores": scores,530 "total": total,531 "avg_seconds": round(sum(r["seconds"] for r in results) / len(results), 2),532 "results": results,533 }534 out = Path(args.out).expanduser()535 out.mkdir(parents=True, exist_ok=True)536 path = out / f"{time.strftime('%Y%m%d-%H%M%S')}-quality-{args.label}.json"537 path.write_text(json.dumps(report, ensure_ascii=False, indent=1))538 print(539 json.dumps(540 {541 "label": args.label,542 "total": total,543 "scores": {c: f"{s['passed']}/{s['total']}" for c, s in scores.items()},544 "avg_seconds": report["avg_seconds"],545 "failed": [f"{r['category']}#{r['i']}" for r in results if not r["ok"]],546 },547 ensure_ascii=False,548 )549 )550 print(f"rapport : {path}")551552553def main() -> None:554 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)555 ap.add_argument("--base", default="http://127.0.0.1:8811")556 ap.add_argument("--model", default="default_model")557 ap.add_argument("--key", default=None)558 ap.add_argument("--label", required=True)559 ap.add_argument("--out", default="~/maclustr-llm-data/bench")560 ap.add_argument("--parallel", type=int, default=4)561 ap.add_argument("--max-tokens", type=int, default=1024)562 ap.add_argument("--extra", default=None)563 asyncio.run(main_async(ap.parse_args()))564565566if __name__ == "__main__":567 main()568