SPB Git

Repository Public

test_limits.py 11.6 KB 302 lines python

1"""Limites, quotas, file d'attente, priorités, annulation, backpressure."""23from __future__ import annotations45import asyncio6import threading7import time8from typing import Any910import fake_engine11import httpx12import pytest13from conftest import MODEL, Stack1415from maclustr_llm.config import EngineSpec16from maclustr_llm.errors import APIError17from maclustr_llm.scheduler import Scheduler181920def body(content: str = "x", **extra: Any) -> dict[str, Any]:21    return {"model": MODEL, "messages": [{"role": "user", "content": content}], **extra}222324def test_requests_per_minute(stack: Stack) -> None:25    _, key = stack.make_key(rpm=3)26    with stack.client(key) as c:27        codes = [c.post("/v1/chat/completions", json=body()).status_code for _ in range(4)]28        assert codes == [200, 200, 200, 429]29        r = c.post("/v1/chat/completions", json=body())30        err = r.json()["error"]31        assert err["type"] == "rate_limit_error" and err["code"] == "rate_limit_exceeded"32        assert 1 <= int(r.headers["retry-after"]) <= 6033        assert r.headers.get("x-request-id")343536def test_tokens_per_minute_and_daily_quota(stack: Stack) -> None:37    _, key = stack.make_key(tpm=150, daily_tokens=0)38    with stack.client(key) as c:39        assert c.post("/v1/chat/completions", json=body("a b c", max_tokens=20)).status_code == 20040        r = c.post("/v1/chat/completions", json=body("a b c", max_tokens=400))41        assert r.status_code == 429 and r.json()["error"]["code"] == "tokens_rate_limit_exceeded"42    _, key = stack.make_key(daily_tokens=5)43    with stack.client(key) as c:44        assert c.post("/v1/chat/completions", json=body("un deux trois quatre cinq six")).status_code == 20045        r = c.post("/v1/chat/completions", json=body())46        assert r.status_code == 429 and r.json()["error"]["code"] == "insufficient_quota"474849def test_quota_survives_restart(stack: Stack) -> None:50    """Le quota quotidien est réamorcé depuis la base : un redémarrage ne le remet pas à zéro."""51    k, key = stack.make_key(daily_tokens=5)52    with stack.client(key) as c:53        assert c.post("/v1/chat/completions", json=body("un deux trois quatre cinq six")).status_code == 20054    stack.run(stack.svc.metrics.flush())55    stack.svc.limiter._states.clear()  # simule un redémarrage du processus56    with stack.client(key) as c:57        assert c.post("/v1/chat/completions", json=body()).status_code == 429585960def _slow_streams(stack: Stack, key: str, n: int, results: list[int], hold: threading.Event) -> list[threading.Thread]:61    def run() -> None:62        with (63            stack.client(key) as c,64            c.stream("POST", "/v1/chat/completions", json=body("[slow] " + "mot " * 20, stream=True)) as r,65        ):66            results.append(r.status_code)67            if r.status_code == 200:68                for _ in r.iter_lines():69                    if hold.is_set():70                        break7172    threads = [threading.Thread(target=run) for _ in range(n)]73    for t in threads:74        t.start()75    return threads767778def test_per_key_concurrency(stack: Stack) -> None:79    _, key = stack.make_key(max_concurrency=1)80    results: list[int] = []81    hold = threading.Event()82    threads = _slow_streams(stack, key, 1, results, hold)83    time.sleep(0.5)84    with stack.client(key) as c:85        r = c.post("/v1/chat/completions", json=body())86        assert r.status_code == 429 and r.json()["error"]["code"] == "concurrency_limit_exceeded"87    hold.set()88    for t in threads:89        t.join(10)909192def test_queue_backpressure_and_recovery(stack: Stack) -> None:93    """Moteur : 2 actives, file : 2. La 5e requête simultanée est refusée tout de suite (503), jamais empilée ;94    quand la charge retombe, le service reprend seul."""95    _, key = stack.make_key(max_concurrency=16, rpm=0)96    results: list[int] = []97    hold = threading.Event()98    threads = _slow_streams(stack, key, 4, results, hold)99    time.sleep(0.7)100    snap = stack.svc.scheduler.snapshot()101    assert snap["active"] == 2 and snap["queued"] == 2102    with stack.client(key) as c:103        r = c.post("/v1/chat/completions", json=body())104        assert r.status_code == 503 and r.json()["error"]["code"] == "engine_overloaded"105        assert r.json()["error"]["type"] == "overloaded_error" and "retry-after" in r.headers106    assert fake_engine.STATE["max_active"] <= 2  # le moteur n'a jamais reçu plus que sa capacité107    hold.set()108    for t in threads:109        t.join(15)110    time.sleep(0.5)111    snap = stack.svc.scheduler.snapshot()112    assert snap["active"] == 0 and snap["queued"] == 0113    with stack.client(key) as c:114        assert c.post("/v1/chat/completions", json=body()).status_code == 200115116117def test_queue_timeout(stack: Stack) -> None:118    _, key = stack.make_key(max_concurrency=16)119    results: list[int] = []120    hold = threading.Event()121    threads = _slow_streams(stack, key, 2, results, hold)122    time.sleep(0.5)123    with stack.client(key) as c:  # file : attente max 3 s dans ce montage, les 2 flux durent ~3,3 s124        t0 = time.time()125        r = c.post("/v1/chat/completions", json=body("[slow] " + "mot " * 40))126        assert r.status_code in (200, 503)127        if r.status_code == 503:128            assert r.json()["error"]["code"] == "queue_timeout" and time.time() - t0 >= 2.9129    hold.set()130    for t in threads:131        t.join(15)132133134def test_client_disconnect_cancels_generation(stack: Stack) -> None:135    """Fermer la connexion arrête la génération côté moteur et libère la place."""136    _, key = stack.make_key()137    with (138        stack.client(key) as c,139        c.stream("POST", "/v1/chat/completions", json=body("[slow] " + "mot " * 200, stream=True)) as r,140    ):141        it = r.iter_lines()142        for _ in range(6):143            next(it)144    deadline = time.time() + 5145    while time.time() < deadline and (stack.svc.scheduler.active or not fake_engine.STATE["cancelled"]):146        time.sleep(0.1)147    assert stack.svc.scheduler.active == 0 and fake_engine.STATE["active"] == 0148    assert fake_engine.STATE["cancelled"] == 1149    stack.run(stack.svc.metrics.flush())150    rows = stack.run(stack.svc.metrics.recent_requests(5))151    assert rows[0]["status"] == 499 and rows[0]["error_code"] == "client_closed_request"152    assert rows[0]["completion_tokens"] > 0  # les tokens déjà produits restent comptabilisés153154155def test_non_stream_disconnect_cancels(stack: Stack) -> None:156    _, key = stack.make_key()157    with stack.client(key) as c, pytest.raises(httpx.ReadTimeout):158        c.post("/v1/chat/completions", json=body("[slow] " + "mot " * 200), timeout=0.6)159    deadline = time.time() + 5160    while time.time() < deadline and fake_engine.STATE["active"]:161        time.sleep(0.1)162    assert fake_engine.STATE["active"] == 0 and stack.svc.scheduler.active == 0163164165# --- ordonnanceur (unitaire) -------------------------------------------------------------------------166def sched(167    max_active: int = 1, max_queue: int = 10, aging: float = 15.0, budget: int = 0, pressure: str = "normal"168) -> Scheduler:169    s = Scheduler(170        (EngineSpec("e", "http://x", ("m",), max_active, budget),), max_queue, 5.0, aging, pressure=lambda: pressure171    )172    s.engines["e"].healthy = True173    return s174175176async def test_priority_order() -> None:177    s = sched()178    first = await s.acquire("m", "client")179    order: list[str] = []180181    async def worker(name: str, prio: str) -> None:182        lease = await s.acquire("m", prio)183        order.append(name)184        s.release(lease)185186    tasks = [187        asyncio.create_task(worker(n, p))188        for n, p in [("fond", "background"), ("client", "client"), ("interne", "internal"), ("admin", "admin")]189    ]190    await asyncio.sleep(0.05)191    s.release(first)192    await asyncio.gather(*tasks)193    assert order == ["admin", "interne", "client", "fond"]194195196async def test_no_starvation_with_aging() -> None:197    """Une requête de fond qui a attendu plus que l'écart de vieillissement passe avant un admin tout neuf."""198    s = sched(aging=0.05)199    first = await s.acquire("m", "admin")200    order: list[str] = []201202    async def worker(name: str, prio: str) -> None:203        lease = await s.acquire("m", prio)204        order.append(name)205        await asyncio.sleep(0.01)206        s.release(lease)207208    old = asyncio.create_task(worker("fond-ancien", "background"))209    await asyncio.sleep(0.3)  # 0,3 s d'attente > 3 rangs × 0,05 s210    new = asyncio.create_task(worker("admin-neuf", "admin"))211    await asyncio.sleep(0.02)212    s.release(first)213    await asyncio.gather(old, new)214    assert order == ["fond-ancien", "admin-neuf"]215216217async def test_queue_full_and_unavailable() -> None:218    s = sched(max_queue=1)219    lease = await s.acquire("m", "client")220    waiter = asyncio.create_task(s.acquire("m", "client"))221    await asyncio.sleep(0.01)222    with pytest.raises(APIError) as exc:223        await s.acquire("m", "admin")224    assert exc.value.status == 503 and exc.value.code == "engine_overloaded" and s.rejected == 1225    s.release(lease)226    s.release(await waiter)227    s.engines["e"].healthy = False228    with pytest.raises(APIError) as exc:229        await s.acquire("m", "client")230    assert exc.value.code == "engine_unavailable"231    with pytest.raises(APIError):232        await s.acquire("inconnu", "client")233234235async def test_cancel_while_queued_releases_nothing_extra() -> None:236    s = sched()237    lease = await s.acquire("m", "client")238    waiter = asyncio.create_task(s.acquire("m", "client"))239    await asyncio.sleep(0.01)240    waiter.cancel()241    with pytest.raises(asyncio.CancelledError):242        await waiter243    assert s.queued == 0244    s.release(lease)245    assert s.active == 0246    s.release(await s.acquire("m", "client"))247248249async def test_token_budget_bounds_memory() -> None:250    """Le budget de tokens en vol borne la mémoire des KV caches : une grosse requête attend, une seule passe toujours."""251    s = sched(max_active=8, budget=10_000)252    big = await s.acquire("m", "client", 9_000)253    small = await s.acquire("m", "client", 900)254    blocked = asyncio.create_task(s.acquire("m", "client", 5_000))255    await asyncio.sleep(0.02)256    assert not blocked.done() and s.engines["e"].tokens == 9_900257    s.release(big)258    lease = await blocked259    assert s.engines["e"].tokens == 5_900260    s.release(small)261    s.release(lease)262    huge = await s.acquire("m", "client", 50_000)  # seule : admise même au-delà du budget263    s.release(huge)264    assert s.engines["e"].tokens == 0265266267async def test_memory_pressure_guard() -> None:268    s = sched(max_active=8, pressure="warn")269    leases = [await s.acquire("m", "client") for _ in range(4)]270    blocked = asyncio.create_task(s.acquire("m", "client"))271    await asyncio.sleep(0.02)272    assert not blocked.done()  # capacité divisée par deux sous pression « warn »273    for lease in leases:274        s.release(lease)275    s.release(await blocked)276    crit = sched(pressure="critical")277    with pytest.raises(APIError) as exc:278        await crit.acquire("m", "admin")279    assert exc.value.code == "memory_pressure"280281282async def test_stream_never_iterated_still_releases(stack: Stack) -> None:283    """Client disparu avant le premier octet du corps : le générateur SSE ne démarre jamais, la place est rendue quand même."""284    from maclustr_llm.inference import _LeasedStream285286    released: list[object] = []287288    async def body():  # type: ignore[no-untyped-def]289        yield b"data: x\n\n"290291    resp = _LeasedStream(released.append, body(), media_type="text/event-stream")292293    async def receive() -> dict[str, str]:294        return {"type": "http.disconnect"}295296    async def send(message: dict[str, object]) -> None:297        raise OSError("client parti")298299    with pytest.raises(BaseException):  # noqa: B017, PT011300        await resp({"type": "http", "asgi": {"spec_version": "2.0"}, "method": "POST", "headers": []}, receive, send)301    assert released == [None]302
Files main
Menu

Theme