SPB Git

Repository Public

quality.py 20.6 KB 568 lines python

1"""Évaluation de qualité à correction automatique, pour comparer des modèles candidats sur le même matériel.23Six catégories (français, anglais, raisonnement, programmation, tool calling, respect des instructions),4chaque item a un vérificateur déterministe : nombre attendu, mots-clés, exécution de tests sur le code produit,5appel d'outil attendu, contrainte de format. Température 0. Ce n'est pas un benchmark académique : c'est un6filtre reproductible pour choisir entre des candidats, complété par les mesures de vitesse de bench.py.78    python bench/quality.py --base http://127.0.0.1:8811 --label instruct-2507-4bit9"""1011from __future__ import annotations1213import argparse14import asyncio15import json16import re17import subprocess18import sys19import tempfile20import time21import unicodedata22from collections.abc import Callable23from pathlib import Path24from typing import Any2526import httpx2728Check = Callable[[dict[str, Any]], bool]293031def norm(text: str) -> str:32    text = unicodedata.normalize("NFKD", text.lower())33    return "".join(c for c in text if not unicodedata.combining(c))343536def has(*words: str) -> Check:37    return lambda r: all(norm(w) in norm(r["content"]) for w in words)383940def has_any(*words: str) -> Check:41    return lambda r: any(norm(w) in norm(r["content"]) for w in words)424344def number(expected: float, tol: float = 1e-6) -> Check:45    def check(r: dict[str, Any]) -> bool:46        m = re.findall(r"(?:reponse|answer)\s*:?\s*\**\s*(-?\d[\d\s.,]*)", norm(r["content"]))47        if not m:48            return False49        raw = m[-1].strip().rstrip(".,").replace(" ", "").replace(" ", "")50        if "," in raw and "." not in raw:51            raw = raw.replace(",", ".")52        raw = raw.replace(",", "")53        try:54            return abs(float(raw) - expected) <= tol55        except ValueError:56            return False5758    return check596061def code(func: str, tests: str) -> Check:62    def check(r: dict[str, Any]) -> bool:63        blocks = re.findall(r"```(?:python|py)?\s*\n(.*?)```", r["content"], re.S)64        src = max(blocks, key=len) if blocks else r["content"]65        if func not in src:66            return False67        with tempfile.TemporaryDirectory() as d:68            path = Path(d) / "sol.py"69            path.write_text(src + "\n\n" + tests + "\nprint('TESTS_OK')\n")70            try:71                out = subprocess.run(72                    [sys.executable, "-I", str(path)], capture_output=True, text=True, timeout=10, cwd=d, check=False73                )74            except subprocess.TimeoutExpired:75                return False76            return "TESTS_OK" in out.stdout7778    return check798081def tool(name: str | None, **args: Any) -> Check:82    def check(r: dict[str, Any]) -> bool:83        calls = r["tool_calls"]84        if name is None:85            return not calls86        for c in calls:87            if c["name"] != name:88                continue89            try:90                got = json.loads(c["arguments"]) if isinstance(c["arguments"], str) else c["arguments"]91            except ValueError:92                return False93            if all(94                norm(str(got.get(k, ""))) == norm(str(v)) or norm(str(v)) in norm(str(got.get(k, "")))95                for k, v in args.items()96            ):97                return True98        return False99100    return check101102103def fmt(pred: Callable[[str], bool]) -> Check:104    return lambda r: pred(r["content"].strip())105106107def is_json_with(*keys: str) -> Callable[[str], bool]:108    def pred(text: str) -> bool:109        text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())110        try:111            obj = json.loads(text)112        except ValueError:113            return False114        return isinstance(obj, dict) and set(obj) == set(keys)115116    return pred117118119TOOLS = [120    {121        "type": "function",122        "function": {123            "name": "get_weather",124            "description": "Météo actuelle d'une ville.",125            "parameters": {126                "type": "object",127                "properties": {128                    "city": {"type": "string"},129                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},130                },131                "required": ["city"],132            },133        },134    },135    {136        "type": "function",137        "function": {138            "name": "search_flights",139            "description": "Recherche des vols entre deux villes.",140            "parameters": {141                "type": "object",142                "properties": {143                    "origin": {"type": "string"},144                    "destination": {"type": "string"},145                    "date": {"type": "string", "description": "AAAA-MM-JJ"},146                },147                "required": ["origin", "destination", "date"],148            },149        },150    },151    {152        "type": "function",153        "function": {154            "name": "convert_currency",155            "description": "Convertit un montant.",156            "parameters": {157                "type": "object",158                "properties": {"amount": {"type": "number"}, "from": {"type": "string"}, "to": {"type": "string"}},159                "required": ["amount", "from", "to"],160            },161        },162    },163]164NUM = " Termine par une ligne « Réponse : <nombre> »."165NUM_EN = ' End with a line "Answer: <number>".'166167ITEMS: list[tuple[str, str, Check, dict[str, Any]]] = [168    # --- français169    ("fr", "Quelle est la capitale du Canada ? Réponds en une phrase.", has("ottawa"), {}),170    (171        "fr",172        "Conjugue le verbe « aller » au subjonctif présent, à la première personne du pluriel. Donne seulement "173        "la forme conjuguée avec son pronom.",174        has("que nous allions"),175        {},176    ),177    (178        "fr",179        "Donne le pluriel de « un cheval », « un bail » et « un festival », séparés par des virgules.",180        has("chevaux", "baux", "festivals"),181        {},182    ),183    (184        "fr",185        'Traduis en français : "The meeting was postponed until next Thursday because of the snowstorm."',186        lambda r: (187            has_any("reportee", "remise", "repoussee", "ajournee")(r)188            and has("jeudi prochain")(r)189            and has_any("tempete de neige")(r)190        ),191        {},192    ),193    (194        "fr",195        "Dans la phrase « Les pommes que j'ai mangées étaient mûres », pourquoi « mangées » s'accorde-t-il ? "196        "Réponds en deux phrases.",197        has_any("cod", "complement d'objet direct", "complement direct"),198        {},199    ),200    (201        "fr",202        "Résume en une seule phrase : « L'entreprise a annoncé mardi la fermeture de son usine de Trois-Rivières, "203        "ce qui entraînera la perte de 320 emplois d'ici la fin de l'année, en raison de la baisse de la demande. »",204        lambda r: has("320")(r) and has_any("trois-rivieres")(r) and len(r["content"]) < 400,205        {},206    ),207    # --- anglais208    ("en", "What is the past participle of 'to write'? Answer in one short sentence.", has("written"), {}),209    (210        "en",211        "Translate into English: « Je n'ai pas eu le temps de finir le rapport avant la réunion. »",212        lambda r: (213            has("report")(r)214            and has("meeting")(r)215            and has_any("didn't have time", "did not have time", "didn't have the time", "did not have the time")(r)216        ),217        {},218    ),219    (220        "en",221        "Which word is the odd one out: apple, banana, carrot, cherry? Answer with the word and a short reason.",222        has("carrot"),223        {},224    ),225    (226        "en",227        'Rewrite in the passive voice: "The committee approved the budget."',228        has("the budget was approved by the committee"),229        {},230    ),231    (232        "en",233        'Summarize in one sentence: "Researchers at the university discovered that the new alloy is 40% lighter '234        'than steel while keeping the same strength, which could cut fuel use in aircraft."',235        lambda r: has("40")(r) and has_any("alloy")(r) and len(r["content"]) < 400,236        {},237    ),238    # --- raisonnement239    (240        "reasoning",241        "Un magasin vend un article 80 $. Il augmente le prix de 25 %, puis accorde un rabais de 20 % sur "242        "le nouveau prix. Quel est le prix final en dollars ?" + NUM,243        number(80),244        {},245    ),246    (247        "reasoning",248        "Trois ouvriers construisent un mur en 12 jours. Combien de jours faut-il à 4 ouvriers travaillant "249        "au même rythme ?" + NUM,250        number(9),251        {},252    ),253    (254        "reasoning",255        "Marie a deux fois l'âge de Paul. Dans 10 ans, la somme de leurs âges sera 50. Quel âge a Marie "256        "aujourd'hui ?" + NUM,257        number(20),258        {},259    ),260    ("reasoning", "Combien y a-t-il de nombres premiers entre 20 et 50 ?" + NUM, number(7), {}),261    (262        "reasoning",263        "Une horloge sonne 6 coups en 5 secondes (intervalles réguliers). Combien de secondes lui "264        "faut-il pour sonner 12 coups ?" + NUM,265        number(11),266        {},267    ),268    (269        "reasoning",270        "A train travels 150 km at 60 km/h, then 100 km at 50 km/h. What is the average speed for the "271        "whole trip in km/h, rounded to one decimal?" + NUM_EN,272        number(55.6, 0.06),273        {},274    ),275    (276        "reasoning",277        "If all bloops are razzies and all razzies are lazzies, are all bloops definitely lazzies? "278        "Answer yes or no, then one sentence.",279        lambda r: norm(r["content"]).lstrip(" *").startswith("yes"),280        {},281    ),282    (283        "reasoning",284        "Dans une course, tu dépasses le coureur qui est en deuxième position. En quelle position es-tu "285        "maintenant ? Réponds par un chiffre." + NUM,286        number(2),287        {},288    ),289    (290        "reasoning",291        "J'ai 3 pommes. J'en mange une, j'en achète 5, puis j'en donne la moitié de ce que j'ai. "292        "Combien m'en reste-t-il ?" + NUM,293        number(3.5, 0.51),294        {},295    ),296    # --- programmation297    (298        "code",299        "Écris une fonction Python `fizzbuzz(n)` qui retourne la liste des chaînes de 1 à n : « Fizz » pour les "300        "multiples de 3, « Buzz » pour ceux de 5, « FizzBuzz » pour les deux, sinon le nombre en chaîne. "301        "Donne uniquement le code dans un bloc ```python.",302        code("def fizzbuzz", "assert fizzbuzz(15)[-1]=='FizzBuzz' and fizzbuzz(5)==['1','2','Fizz','4','Buzz']"),303        {},304    ),305    (306        "code",307        "Write a Python function `merge_intervals(intervals)` that merges overlapping intervals given as a list "308        "of [start, end] lists and returns them sorted. Only the code in a ```python block.",309        code(310            "def merge_intervals",311            "assert [list(x) for x in merge_intervals([[1,3],[2,6],[8,10],[15,18]])]==[[1,6],[8,10],[15,18]]\n"312            "assert [list(x) for x in merge_intervals([[1,4],[4,5]])]==[[1,5]]\nassert merge_intervals([])==[]",313        ),314        {},315    ),316    (317        "code",318        "Write a Python function `is_valid_parentheses(s)` returning True if the string of ()[]{} is balanced. "319        "Only the code in a ```python block.",320        code(321            "def is_valid_parentheses",322            "assert is_valid_parentheses('()[]{}') and is_valid_parentheses('{[]}') and "323            "not is_valid_parentheses('(]') and not is_valid_parentheses('((') and is_valid_parentheses('')",324        ),325        {},326    ),327    (328        "code",329        "Écris une fonction Python `lru(capacity)` qui retourne un objet avec les méthodes `get(key)` (retourne "330        "-1 si absent) et `put(key, value)`, implémentant un cache LRU. Uniquement le code dans un bloc ```python.",331        code(332            "def lru",333            "c=lru(2); c.put(1,1); c.put(2,2); assert c.get(1)==1; c.put(3,3); assert c.get(2)==-1\n"334            "c.put(4,4); assert c.get(1)==-1 and c.get(3)==3 and c.get(4)==4",335        ),336        {},337    ),338    (339        "code",340        "Write a Python function `roman_to_int(s)` converting a Roman numeral to an integer. Only the code in a "341        "```python block.",342        code(343            "def roman_to_int",344            "assert roman_to_int('MCMXCIV')==1994 and roman_to_int('LVIII')==58 and roman_to_int('IX')==9",345        ),346        {},347    ),348    (349        "code",350        "This Python function has a bug. Fix it and return only the corrected code in a ```python block.\n"351        "```python\ndef median(xs):\n    xs.sort()\n    n = len(xs)\n    if n % 2 == 0:\n        return xs[n // 2]\n"352        "    return (xs[n // 2 - 1] + xs[n // 2]) / 2\n```",353        code("def median", "assert median([3,1,2])==2 and median([4,1,3,2])==2.5 and median([5])==5"),354        {},355    ),356    (357        "code",358        "Write a Python function `top_k_words(text, k)` that returns the k most frequent lowercase words as a "359        "list of (word, count) tuples, ties broken alphabetically. Words are sequences of letters. Only the code "360        "in a ```python block.",361        code(362            "def top_k_words",363            "assert [tuple(x) for x in top_k_words('a b b c c c A', 2)]==[('c',3),('a',2)]\n"364            "assert [tuple(x) for x in top_k_words('x y', 5)]==[('x',1),('y',1)]",365        ),366        {},367    ),368    # --- tool calling369    ("tools", "Quel temps fait-il à Québec en ce moment ?", tool("get_weather", city="Québec"), {"tools": TOOLS}),370    (371        "tools",372        "Donne-moi la météo de Tokyo en fahrenheit.",373        tool("get_weather", city="Tokyo", unit="fahrenheit"),374        {"tools": TOOLS},375    ),376    (377        "tools",378        "Trouve-moi un vol de Montréal à Paris le 14 février 2027.",379        tool("search_flights", origin="Montréal", destination="Paris", date="2027-02-14"),380        {"tools": TOOLS},381    ),382    (383        "tools",384        "Combien font 250 dollars canadiens en euros ?",385        tool("convert_currency", amount=250, to="EUR"),386        {"tools": TOOLS},387    ),388    ("tools", "Explique-moi en une phrase ce qu'est une API REST.", tool(None), {"tools": TOOLS}),389    ("tools", "Merci, c'est tout pour aujourd'hui !", tool(None), {"tools": TOOLS}),390    (391        "tools",392        "I'm flying from Toronto to Vancouver on 2027-03-02, can you look for flights?",393        tool("search_flights", origin="Toronto", destination="Vancouver", date="2027-03-02"),394        {"tools": TOOLS},395    ),396    # --- respect des instructions397    (398        "instructions",399        "Réponds par un seul mot, sans ponctuation : quelle est la couleur du ciel par temps clair ?",400        fmt(lambda t: norm(t) in ("bleu", "bleue")),401        {},402    ),403    (404        "instructions",405        'Retourne uniquement un objet JSON avec exactement les clés "nom" et "age" pour une personne '406        "fictive. Aucun autre texte.",407        fmt(is_json_with("nom", "age")),408        {},409    ),410    (411        "instructions",412        "Liste exactement trois fruits, un par ligne, sans numérotation, sans puce et sans autre texte.",413        fmt(414            lambda t: (415                len([x for x in t.splitlines() if x.strip()]) == 3 and not re.search(r"^\s*(?:[-*•]|\d+[.)])", t, re.M)416            )417        ),418        {},419    ),420    (421        "instructions",422        "ANSWER ONLY IN UPPERCASE: what is the largest planet of the solar system?",423        fmt(lambda t: "JUPITER" in t and t == t.upper()),424        {},425    ),426    (427        "instructions",428        "Réponds en anglais uniquement, même si la question est en français : quel est le plus grand océan du monde ?",429        lambda r: has("pacific")(r) and not has_any("pacifique", "l'ocean")(r),430        {},431    ),432    (433        "instructions",434        "Écris une phrase de exactement 7 mots sur l'hiver. Rien d'autre.",435        fmt(lambda t: len(re.findall(r"[\w'’-]+", t.strip().strip('."«» '))) == 7),436        {},437    ),438    (439        "instructions",440        "Tu es un assistant qui ne parle que de cuisine. Si on te demande autre chose, réponds exactement "441        "« Je ne parle que de cuisine. »\n\nQuestion : qui a gagné la Coupe du monde 2018 ?",442        fmt(lambda t: norm(t).strip('"« »').startswith("je ne parle que de cuisine")),443        {},444    ),445]446447448async def ask(449    http: httpx.AsyncClient, base: str, model: str, prompt: str, extra: dict[str, Any], max_tokens: int450) -> dict[str, Any]:451    payload = {452        "model": model,453        "messages": [{"role": "user", "content": prompt}],454        "max_tokens": max_tokens,455        "temperature": 0.0,456        **extra,457    }458    t0 = time.perf_counter()459    r = await http.post(f"{base}/v1/chat/completions", json=payload)460    r.raise_for_status()461    body = r.json()462    msg = body["choices"][0]["message"]463    calls = [464        {"name": c["function"]["name"], "arguments": c["function"]["arguments"]} for c in msg.get("tool_calls") or []465    ]466    return {467        "content": msg.get("content") or "",468        "tool_calls": calls,469        "usage": body.get("usage", {}),470        "seconds": time.perf_counter() - t0,471        "finish": body["choices"][0].get("finish_reason"),472    }473474475async def main_async(args: argparse.Namespace) -> None:476    extra = json.loads(args.extra) if args.extra else {}477    headers = {"Authorization": f"Bearer {args.key}"} if args.key else {}478    results: list[dict[str, Any]] = []479    async with httpx.AsyncClient(timeout=600, headers=headers) as http:480        sem = asyncio.Semaphore(args.parallel)481482        async def one(i: int, cat: str, prompt: str, check: Check, kw: dict[str, Any]) -> None:483            async with sem:484                try:485                    r = await ask(http, args.base, args.model, prompt, {**extra, **kw}, args.max_tokens)486                    ok = bool(check(r))487                except Exception as exc:  # noqa: BLE001 - une panne compte comme un échec mesuré488                    r, ok = (489                        {490                            "content": f"ERREUR {type(exc).__name__}: {exc}",491                            "tool_calls": [],492                            "usage": {},493                            "seconds": 0,494                            "finish": None,495                        },496                        False,497                    )498                results.append(499                    {500                        "i": i,501                        "category": cat,502                        "ok": ok,503                        "prompt": prompt,504                        "answer": r["content"][:1500],505                        "tool_calls": r["tool_calls"],506                        "seconds": round(r["seconds"], 2),507                        "completion_tokens": r["usage"].get("completion_tokens"),508                        "finish": r["finish"],509                    }510                )511512        await asyncio.gather(*(one(i, *item) for i, item in enumerate(ITEMS)))513    results.sort(key=lambda r: r["i"])514    cats: dict[str, list[bool]] = {}515    for r in results:516        cats.setdefault(r["category"], []).append(r["ok"])517    scores = {c: {"passed": sum(v), "total": len(v), "score": round(sum(v) / len(v), 3)} for c, v in cats.items()}518    total = {"passed": sum(r["ok"] for r in results), "total": len(results)}519    total["score"] = round(total["passed"] / total["total"], 3)520    report = {521        "meta": {522            "date": time.strftime("%Y-%m-%d %H:%M:%S"),523            "label": args.label,524            "extra": extra,525            "items": len(ITEMS),526            "temperature": 0.0,527            "max_tokens": args.max_tokens,528        },529        "scores": scores,530        "total": total,531        "avg_seconds": round(sum(r["seconds"] for r in results) / len(results), 2),532        "results": results,533    }534    out = Path(args.out).expanduser()535    out.mkdir(parents=True, exist_ok=True)536    path = out / f"{time.strftime('%Y%m%d-%H%M%S')}-quality-{args.label}.json"537    path.write_text(json.dumps(report, ensure_ascii=False, indent=1))538    print(539        json.dumps(540            {541                "label": args.label,542                "total": total,543                "scores": {c: f"{s['passed']}/{s['total']}" for c, s in scores.items()},544                "avg_seconds": report["avg_seconds"],545                "failed": [f"{r['category']}#{r['i']}" for r in results if not r["ok"]],546            },547            ensure_ascii=False,548        )549    )550    print(f"rapport : {path}")551552553def main() -> None:554    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)555    ap.add_argument("--base", default="http://127.0.0.1:8811")556    ap.add_argument("--model", default="default_model")557    ap.add_argument("--key", default=None)558    ap.add_argument("--label", required=True)559    ap.add_argument("--out", default="~/maclustr-llm-data/bench")560    ap.add_argument("--parallel", type=int, default=4)561    ap.add_argument("--max-tokens", type=int, default=1024)562    ap.add_argument("--extra", default=None)563    asyncio.run(main_async(ap.parse_args()))564565566if __name__ == "__main__":567    main()568
Files main
Menu

Theme