Repository Public
1"""Limites, quotas, file d'attente, priorités, annulation, backpressure."""23from __future__ import annotations45import asyncio6import threading7import time8from typing import Any910import fake_engine11import httpx12import pytest13from conftest import MODEL, Stack1415from maclustr_llm.config import EngineSpec16from maclustr_llm.errors import APIError17from maclustr_llm.scheduler import Scheduler181920def body(content: str = "x", **extra: Any) -> dict[str, Any]:21 return {"model": MODEL, "messages": [{"role": "user", "content": content}], **extra}222324def test_requests_per_minute(stack: Stack) -> None:25 _, key = stack.make_key(rpm=3)26 with stack.client(key) as c:27 codes = [c.post("/v1/chat/completions", json=body()).status_code for _ in range(4)]28 assert codes == [200, 200, 200, 429]29 r = c.post("/v1/chat/completions", json=body())30 err = r.json()["error"]31 assert err["type"] == "rate_limit_error" and err["code"] == "rate_limit_exceeded"32 assert 1 <= int(r.headers["retry-after"]) <= 6033 assert r.headers.get("x-request-id")343536def test_tokens_per_minute_and_daily_quota(stack: Stack) -> None:37 _, key = stack.make_key(tpm=150, daily_tokens=0)38 with stack.client(key) as c:39 assert c.post("/v1/chat/completions", json=body("a b c", max_tokens=20)).status_code == 20040 r = c.post("/v1/chat/completions", json=body("a b c", max_tokens=400))41 assert r.status_code == 429 and r.json()["error"]["code"] == "tokens_rate_limit_exceeded"42 _, key = stack.make_key(daily_tokens=5)43 with stack.client(key) as c:44 assert c.post("/v1/chat/completions", json=body("un deux trois quatre cinq six")).status_code == 20045 r = c.post("/v1/chat/completions", json=body())46 assert r.status_code == 429 and r.json()["error"]["code"] == "insufficient_quota"474849def test_quota_survives_restart(stack: Stack) -> None:50 """Le quota quotidien est réamorcé depuis la base : un redémarrage ne le remet pas à zéro."""51 k, key = stack.make_key(daily_tokens=5)52 with stack.client(key) as c:53 assert c.post("/v1/chat/completions", json=body("un deux trois quatre cinq six")).status_code == 20054 stack.run(stack.svc.metrics.flush())55 stack.svc.limiter._states.clear() # simule un redémarrage du processus56 with stack.client(key) as c:57 assert c.post("/v1/chat/completions", json=body()).status_code == 429585960def _slow_streams(stack: Stack, key: str, n: int, results: list[int], hold: threading.Event) -> list[threading.Thread]:61 def run() -> None:62 with (63 stack.client(key) as c,64 c.stream("POST", "/v1/chat/completions", json=body("[slow] " + "mot " * 20, stream=True)) as r,65 ):66 results.append(r.status_code)67 if r.status_code == 200:68 for _ in r.iter_lines():69 if hold.is_set():70 break7172 threads = [threading.Thread(target=run) for _ in range(n)]73 for t in threads:74 t.start()75 return threads767778def test_per_key_concurrency(stack: Stack) -> None:79 _, key = stack.make_key(max_concurrency=1)80 results: list[int] = []81 hold = threading.Event()82 threads = _slow_streams(stack, key, 1, results, hold)83 time.sleep(0.5)84 with stack.client(key) as c:85 r = c.post("/v1/chat/completions", json=body())86 assert r.status_code == 429 and r.json()["error"]["code"] == "concurrency_limit_exceeded"87 hold.set()88 for t in threads:89 t.join(10)909192def test_queue_backpressure_and_recovery(stack: Stack) -> None:93 """Moteur : 2 actives, file : 2. La 5e requête simultanée est refusée tout de suite (503), jamais empilée ;94 quand la charge retombe, le service reprend seul."""95 _, key = stack.make_key(max_concurrency=16, rpm=0)96 results: list[int] = []97 hold = threading.Event()98 threads = _slow_streams(stack, key, 4, results, hold)99 time.sleep(0.7)100 snap = stack.svc.scheduler.snapshot()101 assert snap["active"] == 2 and snap["queued"] == 2102 with stack.client(key) as c:103 r = c.post("/v1/chat/completions", json=body())104 assert r.status_code == 503 and r.json()["error"]["code"] == "engine_overloaded"105 assert r.json()["error"]["type"] == "overloaded_error" and "retry-after" in r.headers106 assert fake_engine.STATE["max_active"] <= 2 # le moteur n'a jamais reçu plus que sa capacité107 hold.set()108 for t in threads:109 t.join(15)110 time.sleep(0.5)111 snap = stack.svc.scheduler.snapshot()112 assert snap["active"] == 0 and snap["queued"] == 0113 with stack.client(key) as c:114 assert c.post("/v1/chat/completions", json=body()).status_code == 200115116117def test_queue_timeout(stack: Stack) -> None:118 _, key = stack.make_key(max_concurrency=16)119 results: list[int] = []120 hold = threading.Event()121 threads = _slow_streams(stack, key, 2, results, hold)122 time.sleep(0.5)123 with stack.client(key) as c: # file : attente max 3 s dans ce montage, les 2 flux durent ~3,3 s124 t0 = time.time()125 r = c.post("/v1/chat/completions", json=body("[slow] " + "mot " * 40))126 assert r.status_code in (200, 503)127 if r.status_code == 503:128 assert r.json()["error"]["code"] == "queue_timeout" and time.time() - t0 >= 2.9129 hold.set()130 for t in threads:131 t.join(15)132133134def test_client_disconnect_cancels_generation(stack: Stack) -> None:135 """Fermer la connexion arrête la génération côté moteur et libère la place."""136 _, key = stack.make_key()137 with (138 stack.client(key) as c,139 c.stream("POST", "/v1/chat/completions", json=body("[slow] " + "mot " * 200, stream=True)) as r,140 ):141 it = r.iter_lines()142 for _ in range(6):143 next(it)144 deadline = time.time() + 5145 while time.time() < deadline and (stack.svc.scheduler.active or not fake_engine.STATE["cancelled"]):146 time.sleep(0.1)147 assert stack.svc.scheduler.active == 0 and fake_engine.STATE["active"] == 0148 assert fake_engine.STATE["cancelled"] == 1149 stack.run(stack.svc.metrics.flush())150 rows = stack.run(stack.svc.metrics.recent_requests(5))151 assert rows[0]["status"] == 499 and rows[0]["error_code"] == "client_closed_request"152 assert rows[0]["completion_tokens"] > 0 # les tokens déjà produits restent comptabilisés153154155def test_non_stream_disconnect_cancels(stack: Stack) -> None:156 _, key = stack.make_key()157 with stack.client(key) as c, pytest.raises(httpx.ReadTimeout):158 c.post("/v1/chat/completions", json=body("[slow] " + "mot " * 200), timeout=0.6)159 deadline = time.time() + 5160 while time.time() < deadline and fake_engine.STATE["active"]:161 time.sleep(0.1)162 assert fake_engine.STATE["active"] == 0 and stack.svc.scheduler.active == 0163164165# --- ordonnanceur (unitaire) -------------------------------------------------------------------------166def sched(167 max_active: int = 1, max_queue: int = 10, aging: float = 15.0, budget: int = 0, pressure: str = "normal"168) -> Scheduler:169 s = Scheduler(170 (EngineSpec("e", "http://x", ("m",), max_active, budget),), max_queue, 5.0, aging, pressure=lambda: pressure171 )172 s.engines["e"].healthy = True173 return s174175176async def test_priority_order() -> None:177 s = sched()178 first = await s.acquire("m", "client")179 order: list[str] = []180181 async def worker(name: str, prio: str) -> None:182 lease = await s.acquire("m", prio)183 order.append(name)184 s.release(lease)185186 tasks = [187 asyncio.create_task(worker(n, p))188 for n, p in [("fond", "background"), ("client", "client"), ("interne", "internal"), ("admin", "admin")]189 ]190 await asyncio.sleep(0.05)191 s.release(first)192 await asyncio.gather(*tasks)193 assert order == ["admin", "interne", "client", "fond"]194195196async def test_no_starvation_with_aging() -> None:197 """Une requête de fond qui a attendu plus que l'écart de vieillissement passe avant un admin tout neuf."""198 s = sched(aging=0.05)199 first = await s.acquire("m", "admin")200 order: list[str] = []201202 async def worker(name: str, prio: str) -> None:203 lease = await s.acquire("m", prio)204 order.append(name)205 await asyncio.sleep(0.01)206 s.release(lease)207208 old = asyncio.create_task(worker("fond-ancien", "background"))209 await asyncio.sleep(0.3) # 0,3 s d'attente > 3 rangs × 0,05 s210 new = asyncio.create_task(worker("admin-neuf", "admin"))211 await asyncio.sleep(0.02)212 s.release(first)213 await asyncio.gather(old, new)214 assert order == ["fond-ancien", "admin-neuf"]215216217async def test_queue_full_and_unavailable() -> None:218 s = sched(max_queue=1)219 lease = await s.acquire("m", "client")220 waiter = asyncio.create_task(s.acquire("m", "client"))221 await asyncio.sleep(0.01)222 with pytest.raises(APIError) as exc:223 await s.acquire("m", "admin")224 assert exc.value.status == 503 and exc.value.code == "engine_overloaded" and s.rejected == 1225 s.release(lease)226 s.release(await waiter)227 s.engines["e"].healthy = False228 with pytest.raises(APIError) as exc:229 await s.acquire("m", "client")230 assert exc.value.code == "engine_unavailable"231 with pytest.raises(APIError):232 await s.acquire("inconnu", "client")233234235async def test_cancel_while_queued_releases_nothing_extra() -> None:236 s = sched()237 lease = await s.acquire("m", "client")238 waiter = asyncio.create_task(s.acquire("m", "client"))239 await asyncio.sleep(0.01)240 waiter.cancel()241 with pytest.raises(asyncio.CancelledError):242 await waiter243 assert s.queued == 0244 s.release(lease)245 assert s.active == 0246 s.release(await s.acquire("m", "client"))247248249async def test_token_budget_bounds_memory() -> None:250 """Le budget de tokens en vol borne la mémoire des KV caches : une grosse requête attend, une seule passe toujours."""251 s = sched(max_active=8, budget=10_000)252 big = await s.acquire("m", "client", 9_000)253 small = await s.acquire("m", "client", 900)254 blocked = asyncio.create_task(s.acquire("m", "client", 5_000))255 await asyncio.sleep(0.02)256 assert not blocked.done() and s.engines["e"].tokens == 9_900257 s.release(big)258 lease = await blocked259 assert s.engines["e"].tokens == 5_900260 s.release(small)261 s.release(lease)262 huge = await s.acquire("m", "client", 50_000) # seule : admise même au-delà du budget263 s.release(huge)264 assert s.engines["e"].tokens == 0265266267async def test_memory_pressure_guard() -> None:268 s = sched(max_active=8, pressure="warn")269 leases = [await s.acquire("m", "client") for _ in range(4)]270 blocked = asyncio.create_task(s.acquire("m", "client"))271 await asyncio.sleep(0.02)272 assert not blocked.done() # capacité divisée par deux sous pression « warn »273 for lease in leases:274 s.release(lease)275 s.release(await blocked)276 crit = sched(pressure="critical")277 with pytest.raises(APIError) as exc:278 await crit.acquire("m", "admin")279 assert exc.value.code == "memory_pressure"280281282async def test_stream_never_iterated_still_releases(stack: Stack) -> None:283 """Client disparu avant le premier octet du corps : le générateur SSE ne démarre jamais, la place est rendue quand même."""284 from maclustr_llm.inference import _LeasedStream285286 released: list[object] = []287288 async def body(): # type: ignore[no-untyped-def]289 yield b"data: x\n\n"290291 resp = _LeasedStream(released.append, body(), media_type="text/event-stream")292293 async def receive() -> dict[str, str]:294 return {"type": "http.disconnect"}295296 async def send(message: dict[str, object]) -> None:297 raise OSError("client parti")298299 with pytest.raises(BaseException): # noqa: B017, PT011300 await resp({"type": "http", "asgi": {"spec_version": "2.0"}, "method": "POST", "headers": []}, receive, send)301 assert released == [None]302