Repository Public
1{2 "meta": {3 "date": "2026-11-19 10:02:57",4 "label": "qwen3-8b-4bit-nothink",5 "host": "m4mm",6 "chip": "Apple M4 Pro",7 "memory_gb": 48,8 "macos": "26.5.1",9 "mlx": "0.32.4",10 "mlx_lm": "0.32.2",11 "model": "qwen3-8b-4bit",12 "base": "http://127.0.0.1:8821",13 "extra": {14 "chat_template_kwargs": {15 "enable_thinking": false16 }17 },18 "decode_concurrency": 16,19 "prompt_concurrency": 820 },21 "summary": {22 "model": "qwen3-8b-4bit",23 "single": {24 "ttft_ms": 165.6,25 "tokens_per_second": 51.726 },27 "concurrency": [28 {29 "clients": 1,30 "aggregate_tps": 49.1,31 "per_request_tps": 50.9,32 "ttft_p50_ms": 168.6,33 "ttft_p95_ms": 246.9,34 "latency_p95_ms": 5258.8,35 "error_rate": 0.036 },37 {38 "clients": 2,39 "aggregate_tps": 83.6,40 "per_request_tps": 43.4,41 "ttft_p50_ms": 241.0,42 "ttft_p95_ms": 241.7,43 "latency_p95_ms": 6122.0,44 "error_rate": 0.045 },46 {47 "clients": 4,48 "aggregate_tps": 106.2,49 "per_request_tps": 27.5,50 "ttft_p50_ms": 377.3,51 "ttft_p95_ms": 381.8,52 "latency_p95_ms": 9662.2,53 "error_rate": 0.054 },55 {56 "clients": 8,57 "aggregate_tps": 112.9,58 "per_request_tps": 14.5,59 "ttft_p50_ms": 460.7,60 "ttft_p95_ms": 1104.3,61 "latency_p95_ms": 18194.6,62 "error_rate": 0.063 },64 {65 "clients": 16,66 "aggregate_tps": 143.6,67 "per_request_tps": 9.6,68 "ttft_p50_ms": 1748.4,69 "ttft_p95_ms": 2642.2,70 "latency_p95_ms": 28963.6,71 "error_rate": 0.072 }73 ],74 "peak_memory_bytes": 9078233642,75 "recommended": {76 "max_active": 1677 }78 },79 "single": {80 "ttft_ms": 165.6,81 "tokens_per_second": 51.782 },83 "scenarios": [84 {85 "name": "short_prompt",86 "description": "Question courte, réponse de 128 tokens.",87 "requests": 5,88 "error_rate": 0.0,89 "ttft_ms_p50": 165.6,90 "ttft_ms_p95": 166.7,91 "ttft_ms_p99": 166.7,92 "latency_ms_p50": 2097.5,93 "latency_ms_p95": 2391.3,94 "latency_ms_p99": 2391.3,95 "tokens_per_second_p50": 51.7,96 "prompt_tokens_avg": 34,97 "completion_tokens_avg": 100,98 "cached_tokens_avg": 13,99 "cache_hit_ratio": 0.371,100 "errors": []101 },102 {103 "name": "coding",104 "description": "Programmation : fonction Python avec tests.",105 "requests": 3,106 "error_rate": 0.0,107 "ttft_ms_p50": 137.9,108 "ttft_ms_p95": 246.8,109 "ttft_ms_p99": 246.8,110 "latency_ms_p50": 10241.9,111 "latency_ms_p95": 10394.7,112 "latency_ms_p99": 10394.7,113 "tokens_per_second_p50": 50.6,114 "prompt_tokens_avg": 50,115 "completion_tokens_avg": 512,116 "cached_tokens_avg": 28,117 "cache_hit_ratio": 0.567,118 "errors": []119 },120 {121 "name": "reasoning",122 "description": "Raisonnement en plusieurs étapes.",123 "requests": 3,124 "error_rate": 0.0,125 "ttft_ms_p50": 249.8,126 "ttft_ms_p95": 249.9,127 "ttft_ms_p99": 249.9,128 "latency_ms_p50": 8120.4,129 "latency_ms_p95": 8127.3,130 "latency_ms_p99": 8127.3,131 "tokens_per_second_p50": 50.7,132 "prompt_tokens_avg": 64,133 "completion_tokens_avg": 400,134 "cached_tokens_avg": 6,135 "cache_hit_ratio": 0.098,136 "errors": []137 },138 {139 "name": "tool_calling",140 "description": "Appel d'outil (get_weather) attendu.",141 "requests": 3,142 "error_rate": 0.0,143 "ttft_ms_p50": 680.1,144 "ttft_ms_p95": 927.2,145 "ttft_ms_p99": 927.2,146 "latency_ms_p50": 699.4,147 "latency_ms_p95": 946.6,148 "latency_ms_p99": 946.6,149 "tokens_per_second_p50": null,150 "prompt_tokens_avg": 185,151 "completion_tokens_avg": 28,152 "cached_tokens_avg": 131,153 "cache_hit_ratio": 0.705,154 "errors": [],155 "tool_call_rate": 1.0156 },157 {158 "name": "multi_turn",159 "description": "Conversation de 5 tours (préfixe réutilisé d'un tour à l'autre).",160 "requests": 5,161 "error_rate": 0.0,162 "ttft_ms_p50": 339.1,163 "ttft_ms_p95": 421.5,164 "ttft_ms_p99": 421.5,165 "latency_ms_p50": 2186.1,166 "latency_ms_p95": 2494.0,167 "latency_ms_p99": 2494.0,168 "tokens_per_second_p50": 51.2,169 "prompt_tokens_avg": 245,170 "completion_tokens_avg": 92,171 "cached_tokens_avg": 150,172 "cache_hit_ratio": 0.613,173 "errors": []174 },175 {176 "name": "repeated_prefix",177 "description": "Préfixe système commun d'environ 3 000 tokens, 5 questions différentes après un premier appel à froid.",178 "requests": 5,179 "error_rate": 0.0,180 "ttft_ms_p50": 210.7,181 "ttft_ms_p95": 213.4,182 "ttft_ms_p99": 213.4,183 "latency_ms_p50": 608.3,184 "latency_ms_p95": 610.5,185 "latency_ms_p99": 610.5,186 "tokens_per_second_p50": 47.9,187 "prompt_tokens_avg": 2945,188 "completion_tokens_avg": 19,189 "cached_tokens_avg": 2925,190 "cache_hit_ratio": 0.993,191 "errors": []192 }193 ],194 "prefix_cache": {195 "cold_ttft_ms": 8420.8,196 "warm_ttft_ms": 210.7,197 "warm_cached_tokens": 2925,198 "prompt_tokens": 2945199 },200 "concurrency": [201 {202 "clients": 1,203 "aggregate_tps": 49.1,204 "per_request_tps": 50.9,205 "ttft_p50_ms": 168.6,206 "ttft_p95_ms": 246.9,207 "latency_p50_ms": 5179.3,208 "latency_p95_ms": 5258.8,209 "latency_p99_ms": 5258.8,210 "error_rate": 0.0,211 "wall_s": 10.44,212 "mlx_peak_bytes": 9023424723,213 "mlx_active_bytes": 7818881090,214 "memory_before": {215 "pressure_level": "normal",216 "free_percent": 79,217 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"218 },219 "memory_after": {220 "pressure_level": "normal",221 "free_percent": 80,222 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"223 },224 "errors": []225 },226 {227 "clients": 2,228 "aggregate_tps": 83.6,229 "per_request_tps": 43.4,230 "ttft_p50_ms": 241.0,231 "ttft_p95_ms": 241.7,232 "latency_p50_ms": 6119.4,233 "latency_p95_ms": 6122.0,234 "latency_p99_ms": 6122.0,235 "error_rate": 0.0,236 "wall_s": 12.24,237 "mlx_peak_bytes": 9023424723,238 "mlx_active_bytes": 8109844546,239 "memory_before": {240 "pressure_level": "normal",241 "free_percent": 80,242 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"243 },244 "memory_after": {245 "pressure_level": "normal",246 "free_percent": 79,247 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"248 },249 "errors": []250 },251 {252 "clients": 4,253 "aggregate_tps": 106.2,254 "per_request_tps": 27.5,255 "ttft_p50_ms": 377.3,256 "ttft_p95_ms": 381.8,257 "latency_p50_ms": 9624.4,258 "latency_p95_ms": 9662.2,259 "latency_p99_ms": 9662.2,260 "error_rate": 0.0,261 "wall_s": 19.29,262 "mlx_peak_bytes": 9023424723,263 "mlx_active_bytes": 7244015686,264 "memory_before": {265 "pressure_level": "normal",266 "free_percent": 79,267 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"268 },269 "memory_after": {270 "pressure_level": "normal",271 "free_percent": 79,272 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"273 },274 "errors": []275 },276 {277 "clients": 8,278 "aggregate_tps": 112.9,279 "per_request_tps": 14.5,280 "ttft_p50_ms": 460.7,281 "ttft_p95_ms": 1104.3,282 "latency_p50_ms": 17988.5,283 "latency_p95_ms": 18194.6,284 "latency_p99_ms": 18194.9,285 "error_rate": 0.0,286 "wall_s": 36.27,287 "mlx_peak_bytes": 9023424723,288 "mlx_active_bytes": 6939043914,289 "memory_before": {290 "pressure_level": "normal",291 "free_percent": 79,292 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"293 },294 "memory_after": {295 "pressure_level": "normal",296 "free_percent": 80,297 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"298 },299 "errors": []300 },301 {302 "clients": 16,303 "aggregate_tps": 143.6,304 "per_request_tps": 9.6,305 "ttft_p50_ms": 1748.4,306 "ttft_p95_ms": 2642.2,307 "latency_p50_ms": 28700.0,308 "latency_p95_ms": 28963.6,309 "latency_p99_ms": 28990.6,310 "error_rate": 0.0,311 "wall_s": 57.06,312 "mlx_peak_bytes": 9078233642,313 "mlx_active_bytes": 7188015170,314 "memory_before": {315 "pressure_level": "normal",316 "free_percent": 80,317 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"318 },319 "memory_after": {320 "pressure_level": "normal",321 "free_percent": 79,322 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"323 },324 "errors": []325 }326 ],327 "peak_memory_bytes": 9078233642,328 "engine_start": {329 "ready": true,330 "requests": 42,331 "errors": 0,332 "model": "qwen3-8b-4bit",333 "load_s": null,334 "warmup_s": 1.6,335 "healthy": true,336 "uptime_s": 42,337 "pid": 81793,338 "mlx": {339 "active_bytes": 7263004738,340 "peak_bytes": 9023424723,341 "cache_bytes": 3679133342 },343 "prompt_cache": {344 "entries": 16,345 "bytes": 479674368346 },347 "decode_concurrency": 16,348 "prompt_concurrency": 8,349 "versions": {350 "mlx": "0.32.4",351 "mlx_lm": "0.32.2"352 }353 },354 "engine_end": {355 "ready": true,356 "requests": 129,357 "errors": 0,358 "model": "qwen3-8b-4bit",359 "load_s": null,360 "warmup_s": 1.6,361 "healthy": true,362 "uptime_s": 269,363 "pid": 81793,364 "mlx": {365 "active_bytes": 7188015170,366 "peak_bytes": 9078233642,367 "cache_bytes": 1283699148368 },369 "prompt_cache": {370 "entries": 16,371 "bytes": 712507392372 },373 "decode_concurrency": 16,374 "prompt_concurrency": 8,375 "versions": {376 "mlx": "0.32.4",377 "mlx_lm": "0.32.2"378 }379 },380 "memory_end": {381 "pressure_level": "normal",382 "free_percent": 79,383 "swap": "total = 0,00M used = 0,00M free = 0,00M (encrypted)"384 },385 "samples_single": [386 {387 "ok": true,388 "ttft_ms": 166.682458948344,389 "latency_ms": 1942.3599999863654,390 "gen_ms": 1756.3439160585403,391 "prompt_tokens": 34,392 "completion_tokens": 92,393 "cached_tokens": 3,394 "finish_reason": "stop",395 "tool_calls": 0,396 "error": null,397 "text": null398 },399 {400 "ok": true,401 "ttft_ms": 164.26812508143485,402 "latency_ms": 1943.8748331740499,403 "gen_ms": 1760.3515828959644,404 "prompt_tokens": 34,405 "completion_tokens": 92,406 "cached_tokens": 15,407 "finish_reason": "stop",408 "tool_calls": 0,409 "error": null,410 "text": null411 },412 {413 "ok": true,414 "ttft_ms": 165.61733302660286,415 "latency_ms": 2098.1041251216084,416 "gen_ms": 1913.2177089340985,417 "prompt_tokens": 34,418 "completion_tokens": 100,419 "cached_tokens": 15,420 "finish_reason": "stop",421 "tool_calls": 0,422 "error": null,423 "text": null424 },425 {426 "ok": true,427 "ttft_ms": 165.38320807740092,428 "latency_ms": 2391.261500073597,429 "gen_ms": 2206.436875043437,430 "prompt_tokens": 34,431 "completion_tokens": 115,432 "cached_tokens": 15,433 "finish_reason": "stop",434 "tool_calls": 0,435 "error": null,436 "text": null437 },438 {439 "ok": true,440 "ttft_ms": 166.21841699816287,441 "latency_ms": 2097.497459035367,442 "gen_ms": 1911.7414171341807,443 "prompt_tokens": 34,444 "completion_tokens": 100,445 "cached_tokens": 15,446 "finish_reason": "stop",447 "tool_calls": 0,448 "error": null,449 "text": null450 }451 ]452}