Repository Public
1{2 "meta": {3 "date": "2026-11-19 10:24:14",4 "label": "production-gateway",5 "host": "m4mm",6 "chip": "Apple M4 Pro",7 "memory_gb": 48,8 "macos": "26.5.1",9 "mlx": "0.32.4",10 "mlx_lm": "0.32.2",11 "model": "qwen3-30b-a3b-instruct-2507-4bit",12 "base": "http://127.0.0.1:8810",13 "extra": {},14 "decode_concurrency": 8,15 "prompt_concurrency": 416 },17 "summary": {18 "model": "qwen3-30b-a3b-instruct-2507-4bit",19 "single": {20 "ttft_ms": 110.7,21 "tokens_per_second": 80.622 },23 "concurrency": [24 {25 "clients": 1,26 "aggregate_tps": 75.1,27 "per_request_tps": 78.7,28 "ttft_p50_ms": 145.9,29 "ttft_p95_ms": 204.9,30 "latency_p95_ms": 3427.6,31 "error_rate": 0.032 },33 {34 "clients": 2,35 "aggregate_tps": 102.8,36 "per_request_tps": 53.4,37 "ttft_p50_ms": 194.4,38 "ttft_p95_ms": 196.3,39 "latency_p95_ms": 4979.2,40 "error_rate": 0.041 },42 {43 "clients": 4,44 "aggregate_tps": 125.6,45 "per_request_tps": 32.6,46 "ttft_p50_ms": 321.6,47 "ttft_p95_ms": 330.2,48 "latency_p95_ms": 8155.2,49 "error_rate": 0.050 },51 {52 "clients": 8,53 "aggregate_tps": 155.8,54 "per_request_tps": 20.4,55 "ttft_p50_ms": 547.5,56 "ttft_p95_ms": 695.1,57 "latency_p95_ms": 13278.9,58 "error_rate": 0.059 },60 {61 "clients": 16,62 "aggregate_tps": 156.8,63 "per_request_tps": 20.5,64 "ttft_p50_ms": 13280.4,65 "ttft_p95_ms": 13924.6,66 "latency_p95_ms": 26152.4,67 "error_rate": 0.068 }69 ],70 "peak_memory_bytes": 20746294644,71 "recommended": {72 "max_active": 1673 }74 },75 "single": {76 "ttft_ms": 110.7,77 "tokens_per_second": 80.678 },79 "scenarios": [80 {81 "name": "short_prompt",82 "description": "Question courte, réponse de 128 tokens.",83 "requests": 5,84 "error_rate": 0.0,85 "ttft_ms_p50": 110.7,86 "ttft_ms_p95": 146.3,87 "ttft_ms_p99": 146.3,88 "latency_ms_p50": 1687.1,89 "latency_ms_p95": 1721.0,90 "latency_ms_p99": 1721.0,91 "tokens_per_second_p50": 80.6,92 "prompt_tokens_avg": 30,93 "completion_tokens_avg": 128,94 "cached_tokens_avg": 13,95 "cache_hit_ratio": 0.42,96 "errors": []97 },98 {99 "name": "long_generation",100 "description": "Génération longue de 1 024 tokens.",101 "requests": 3,102 "error_rate": 0.0,103 "ttft_ms_p50": 140.9,104 "ttft_ms_p95": 187.9,105 "ttft_ms_p99": 187.9,106 "latency_ms_p50": 13539.1,107 "latency_ms_p95": 13590.7,108 "latency_ms_p99": 13590.7,109 "tokens_per_second_p50": 76.3,110 "prompt_tokens_avg": 40,111 "completion_tokens_avg": 1024,112 "cached_tokens_avg": 12,113 "cache_hit_ratio": 0.292,114 "errors": []115 },116 {117 "name": "long_prompt",118 "description": "Prompt d'environ 4 000 tokens (jamais vu), réponse de 128 tokens.",119 "requests": 3,120 "error_rate": 0.0,121 "ttft_ms_p50": 4646.0,122 "ttft_ms_p95": 4650.4,123 "ttft_ms_p99": 4650.4,124 "latency_ms_p50": 6645.0,125 "latency_ms_p95": 6653.1,126 "latency_ms_p99": 6653.1,127 "tokens_per_second_p50": 63.4,128 "prompt_tokens_avg": 3042,129 "completion_tokens_avg": 128,130 "cached_tokens_avg": 4,131 "cache_hit_ratio": 0.001,132 "errors": []133 },134 {135 "name": "large_context",136 "description": "Contexte d'environ 16 000 tokens (jamais vu), réponse de 128 tokens.",137 "requests": 2,138 "error_rate": 0.0,139 "ttft_ms_p50": 27428.6,140 "ttft_ms_p95": 27637.8,141 "ttft_ms_p99": 27637.8,142 "latency_ms_p50": 30394.3,143 "latency_ms_p95": 30507.2,144 "latency_ms_p99": 30507.2,145 "tokens_per_second_p50": 42.9,146 "prompt_tokens_avg": 12275,147 "completion_tokens_avg": 126,148 "cached_tokens_avg": 4,149 "cache_hit_ratio": 0.0,150 "errors": []151 },152 {153 "name": "coding",154 "description": "Programmation : fonction Python avec tests.",155 "requests": 3,156 "error_rate": 0.0,157 "ttft_ms_p50": 83.2,158 "ttft_ms_p95": 338.3,159 "ttft_ms_p99": 338.3,160 "latency_ms_p50": 6640.3,161 "latency_ms_p95": 6885.3,162 "latency_ms_p99": 6885.3,163 "tokens_per_second_p50": 78.1,164 "prompt_tokens_avg": 46,165 "completion_tokens_avg": 512,166 "cached_tokens_avg": 27,167 "cache_hit_ratio": 0.587,168 "errors": []169 },170 {171 "name": "reasoning",172 "description": "Raisonnement en plusieurs étapes.",173 "requests": 3,174 "error_rate": 0.0,175 "ttft_ms_p50": 242.7,176 "ttft_ms_p95": 256.3,177 "ttft_ms_p99": 256.3,178 "latency_ms_p50": 5339.9,179 "latency_ms_p95": 5344.1,180 "latency_ms_p99": 5344.1,181 "tokens_per_second_p50": 78.2,182 "prompt_tokens_avg": 60,183 "completion_tokens_avg": 400,184 "cached_tokens_avg": 6,185 "cache_hit_ratio": 0.105,186 "errors": []187 },188 {189 "name": "tool_calling",190 "description": "Appel d'outil (get_weather) attendu.",191 "requests": 3,192 "error_rate": 0.0,193 "ttft_ms_p50": 453.3,194 "ttft_ms_p95": 586.2,195 "ttft_ms_p99": 586.2,196 "latency_ms_p50": 465.3,197 "latency_ms_p95": 598.1,198 "latency_ms_p99": 598.1,199 "tokens_per_second_p50": null,200 "prompt_tokens_avg": 181,201 "completion_tokens_avg": 23,202 "cached_tokens_avg": 112,203 "cache_hit_ratio": 0.616,204 "errors": [],205 "tool_call_rate": 1.0206 },207 {208 "name": "multi_turn",209 "description": "Conversation de 5 tours (préfixe réutilisé d'un tour à l'autre).",210 "requests": 5,211 "error_rate": 0.0,212 "ttft_ms_p50": 118.9,213 "ttft_ms_p95": 170.0,214 "ttft_ms_p99": 170.0,215 "latency_ms_p50": 1937.2,216 "latency_ms_p95": 2136.6,217 "latency_ms_p99": 2136.6,218 "tokens_per_second_p50": 77.8,219 "prompt_tokens_avg": 343,220 "completion_tokens_avg": 143,221 "cached_tokens_avg": 325,222 "cache_hit_ratio": 0.948,223 "errors": []224 },225 {226 "name": "repeated_prefix",227 "description": "Préfixe système commun d'environ 3 000 tokens, 5 questions différentes après un premier appel à froid.",228 "requests": 5,229 "error_rate": 0.0,230 "ttft_ms_p50": 147.2,231 "ttft_ms_p95": 149.5,232 "ttft_ms_p99": 149.5,233 "latency_ms_p50": 391.1,234 "latency_ms_p95": 391.7,235 "latency_ms_p99": 391.7,236 "tokens_per_second_p50": 69.9,237 "prompt_tokens_avg": 2304,238 "completion_tokens_avg": 17,239 "cached_tokens_avg": 2288,240 "cache_hit_ratio": 0.993,241 "errors": []242 }243 ],244 "prefix_cache": {245 "cold_ttft_ms": 3564.2,246 "warm_ttft_ms": 147.2,247 "warm_cached_tokens": 2288,248 "prompt_tokens": 2304249 },250 "concurrency": [251 {252 "clients": 1,253 "aggregate_tps": 75.1,254 "per_request_tps": 78.7,255 "ttft_p50_ms": 145.9,256 "ttft_p95_ms": 204.9,257 "latency_p50_ms": 3387.3,258 "latency_p95_ms": 3427.6,259 "latency_p99_ms": 3427.6,260 "error_rate": 0.0,261 "wall_s": 6.82,262 "mlx_peak_bytes": 20746294644,263 "mlx_active_bytes": 18691207234,264 "memory_before": {265 "pressure_level": "normal",266 "free_percent": 58,267 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"268 },269 "memory_after": {270 "pressure_level": "normal",271 "free_percent": 58,272 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"273 },274 "errors": []275 },276 {277 "clients": 2,278 "aggregate_tps": 102.8,279 "per_request_tps": 53.4,280 "ttft_p50_ms": 194.4,281 "ttft_p95_ms": 196.3,282 "latency_p50_ms": 4976.4,283 "latency_p95_ms": 4979.2,284 "latency_p99_ms": 4979.2,285 "error_rate": 0.0,286 "wall_s": 9.96,287 "mlx_peak_bytes": 20746294644,288 "mlx_active_bytes": 18078101570,289 "memory_before": {290 "pressure_level": "normal",291 "free_percent": 58,292 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"293 },294 "memory_after": {295 "pressure_level": "normal",296 "free_percent": 59,297 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"298 },299 "errors": []300 },301 {302 "clients": 4,303 "aggregate_tps": 125.6,304 "per_request_tps": 32.6,305 "ttft_p50_ms": 321.6,306 "ttft_p95_ms": 330.2,307 "latency_p50_ms": 8140.3,308 "latency_p95_ms": 8155.2,309 "latency_p99_ms": 8155.2,310 "error_rate": 0.0,311 "wall_s": 16.3,312 "mlx_peak_bytes": 20746294644,313 "mlx_active_bytes": 17890635846,314 "memory_before": {315 "pressure_level": "normal",316 "free_percent": 59,317 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"318 },319 "memory_after": {320 "pressure_level": "normal",321 "free_percent": 59,322 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"323 },324 "errors": []325 },326 {327 "clients": 8,328 "aggregate_tps": 155.8,329 "per_request_tps": 20.4,330 "ttft_p50_ms": 547.5,331 "ttft_p95_ms": 695.1,332 "latency_p50_ms": 13180.9,333 "latency_p95_ms": 13278.9,334 "latency_p99_ms": 13299.4,335 "error_rate": 0.0,336 "wall_s": 26.28,337 "mlx_peak_bytes": 20746294644,338 "mlx_active_bytes": 17915785294,339 "memory_before": {340 "pressure_level": "normal",341 "free_percent": 59,342 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"343 },344 "memory_after": {345 "pressure_level": "normal",346 "free_percent": 58,347 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"348 },349 "errors": []350 },351 {352 "clients": 16,353 "aggregate_tps": 156.8,354 "per_request_tps": 20.5,355 "ttft_p50_ms": 13280.4,356 "ttft_p95_ms": 13924.6,357 "latency_p50_ms": 25675.0,358 "latency_p95_ms": 26152.4,359 "latency_p99_ms": 26152.7,360 "error_rate": 0.0,361 "wall_s": 52.25,362 "mlx_peak_bytes": 20746294644,363 "mlx_active_bytes": 17911984206,364 "memory_before": {365 "pressure_level": "normal",366 "free_percent": 58,367 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"368 },369 "memory_after": {370 "pressure_level": "normal",371 "free_percent": 59,372 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"373 },374 "errors": []375 }376 ],377 "peak_memory_bytes": 20746294644,378 "engine_start": {379 "ready": true,380 "requests": 1,381 "errors": 0,382 "model": "qwen3-30b-a3b-instruct-2507-4bit",383 "load_s": null,384 "warmup_s": 0.91,385 "healthy": true,386 "uptime_s": 7,387 "pid": 93140,388 "mlx": {389 "active_bytes": 17238159426,390 "peak_bytes": 17239718164,391 "cache_bytes": 1684360392 },393 "prompt_cache": {394 "entries": 2,395 "bytes": 4030464396 },397 "decode_concurrency": 8,398 "prompt_concurrency": 4,399 "versions": {400 "mlx": "0.32.4",401 "mlx_lm": "0.32.2"402 }403 },404 "engine_end": {405 "ready": true,406 "requests": 96,407 "errors": 0,408 "model": "qwen3-30b-a3b-instruct-2507-4bit",409 "load_s": null,410 "warmup_s": 0.91,411 "healthy": true,412 "uptime_s": 303,413 "pid": 93140,414 "mlx": {415 "active_bytes": 17911984206,416 "peak_bytes": 20746294644,417 "cache_bytes": 753521749418 },419 "prompt_cache": {420 "entries": 8,421 "bytes": 234356736422 },423 "decode_concurrency": 8,424 "prompt_concurrency": 4,425 "versions": {426 "mlx": "0.32.4",427 "mlx_lm": "0.32.2"428 }429 },430 "memory_end": {431 "pressure_level": "normal",432 "free_percent": 59,433 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"434 },435 "samples_single": [436 {437 "ok": true,438 "ttft_ms": 146.3294168934226,439 "latency_ms": 1720.992041984573,440 "gen_ms": 1574.0205829497427,441 "prompt_tokens": 30,442 "completion_tokens": 128,443 "cached_tokens": 3,444 "finish_reason": "length",445 "tool_calls": 0,446 "error": null,447 "text": null448 },449 {450 "ok": true,451 "ttft_ms": 110.72700005024672,452 "latency_ms": 1695.9163749124855,453 "gen_ms": 1584.5394579228014,454 "prompt_tokens": 30,455 "completion_tokens": 128,456 "cached_tokens": 15,457 "finish_reason": "length",458 "tool_calls": 0,459 "error": null,460 "text": null461 },462 {463 "ok": true,464 "ttft_ms": 108.60166605561972,465 "latency_ms": 1684.7845001611859,466 "gen_ms": 1575.3610839601606,467 "prompt_tokens": 30,468 "completion_tokens": 128,469 "cached_tokens": 15,470 "finish_reason": "length",471 "tool_calls": 0,472 "error": null,473 "text": null474 },475 {476 "ok": true,477 "ttft_ms": 110.79249996691942,478 "latency_ms": 1687.1115830726922,479 "gen_ms": 1575.5937080830336,480 "prompt_tokens": 30,481 "completion_tokens": 128,482 "cached_tokens": 15,483 "finish_reason": "length",484 "tool_calls": 0,485 "error": null,486 "text": null487 },488 {489 "ok": true,490 "ttft_ms": 108.75791613943875,491 "latency_ms": 1685.490250121802,492 "gen_ms": 1576.0358748957515,493 "prompt_tokens": 30,494 "completion_tokens": 128,495 "cached_tokens": 15,496 "finish_reason": "length",497 "tool_calls": 0,498 "error": null,499 "text": null500 }501 ]502}