"""Cheap deliberate-error smoke tests for the OpenAI API (all requests are rejected before any generation => $0). Also unit-tests examples/shared/errors/classify_error.py against bodies captured live on 2026-09-18. Run: .venv/bin/pytest -q tests/openai/test_errors.py """ from __future__ import annotations import sys from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "examples/shared/errors")) from classify_error import classify # noqa: E402 def _err(body): assert isinstance(body, dict) and "error" in body, body return body["error"] def test_invalid_api_key_401(openai): st, body, _ = openai("GET", "/v1/models", extra_headers={"Authorization": "Bearer sk-invalid"}, note="test_errors invalid key") assert st == 401 e = _err(body) assert e["code"] == "invalid_api_key" and e["type"] == "invalid_request_error" assert classify("openai", st, body).category == "auth" def test_model_not_found_404(openai): st, body, _ = openai("POST", "/v1/responses", {"model": "does-not-exist-model", "input": "Reply with OK.", "max_output_tokens": 16}, note="test_errors model_not_found") assert st == 404 and _err(body)["code"] == "model_not_found" c = classify("openai", st, body) assert c.category == "not_found" and not c.retryable def test_invalid_type_400(openai): st, body, _ = openai("POST", "/v1/responses", {"model": "gpt-5.4-nano", "input": "Reply with OK.", "max_output_tokens": "sixteen"}, note="test_errors invalid_type") assert st == 400 and _err(body)["code"] == "invalid_type" and _err(body)["param"] == "max_output_tokens" def test_missing_required_parameter_400(openai): st, body, _ = openai("POST", "/v1/responses", {"input": "Reply with OK."}, note="test_errors missing model") assert st == 400 and _err(body)["code"] == "missing_required_parameter" and _err(body)["param"] == "model" def test_invalid_json_400(openai): st, body, _ = openai("POST", "/v1/responses", data=b'{"model": ', note="test_errors invalid json") assert st == 400 and _err(body)["code"] == "invalid_json" def test_unknown_url_404_has_empty_body(openai): st, body, hdrs = openai("GET", "/v1/this_endpoint_does_not_exist", note="test_errors unknown url") assert st == 404 assert body in (b"", "", None) or body == b"" assert not any(k.lower() == "x-request-id" for k in hdrs) # edge 404: no request id assert classify("openai", st, body, hdrs).category == "not_found" def test_response_not_found_404(openai): st, body, _ = openai("GET", "/v1/responses/resp_000000000000000000000000000000", note="test_errors response 404") assert st == 404 and _err(body)["type"] == "invalid_request_error" def test_admin_scope_403_bare_string(openai): """Admin endpoints answer 403 with `error` as a bare string when the key lacks api.management.read.""" st, body, _ = openai("GET", "/v1/organization/projects?limit=1", note="test_errors admin 403 shape") if st == 200: pytest.skip("an Admin key is configured; restriction not reproducible") assert st == 403 and isinstance(body["error"], str) and "Missing scopes" in body["error"] c = classify("openai", st, body) assert c.category == "permission" and not c.retryable # ---------------- offline classifier matrix (captured shapes) ---------------- @pytest.mark.parametrize("status,body,category,retryable", [ (401, {"error": {"message": "Incorrect API key provided", "type": "invalid_request_error", "param": None, "code": "invalid_api_key"}}, "auth", False), (403, {"error": "You have insufficient permissions for this operation. Missing scopes: api.management.read."}, "permission", False), (403, {"error": {"message": "blocked", "type": "invalid_request_error", "param": None, "code": "misalignment_policy_violation"}}, "safety", False), (404, {"error": {"message": "The model `x` does not exist", "type": "invalid_request_error", "param": None, "code": "model_not_found"}}, "not_found", False), (404, b"", "not_found", False), (400, {"error": {"message": "Unknown parameter: 'x'.", "type": "invalid_request_error", "param": "x", "code": "unknown_parameter"}}, "bad_request", False), (422, {"error": {"message": "unprocessable", "type": "invalid_request_error", "param": None, "code": None}}, "bad_request", True), (429, {"error": {"message": "Rate limit reached", "type": "rate_limit_error", "param": None, "code": "rate_limit_exceeded"}}, "rate_limit", True), (429, {"error": {"message": "slow down", "type": "rate_limit_error", "param": None, "code": "slow_down"}}, "rate_limit", True), (429, {"error": {"message": "quota", "type": "insufficient_quota", "param": None, "code": "insufficient_quota"}}, "quota_billing", False), (429, {"error": {"message": "spend", "type": "insufficient_quota", "param": None, "code": "project_spend_limit_exceeded"}}, "quota_billing", False), (500, {"error": {"message": "server had an error", "type": "server_error", "param": None, "code": None}}, "server", True), (503, {"error": {"message": "overloaded", "type": "service_unavailable_error", "param": None, "code": "server_is_overloaded"}}, "server", True), (None, None, "network", True), ]) def test_classifier_matrix(status, body, category, retryable): c = classify("openai", status, body, {"retry-after": "3", "x-request-id": "req_x"}) assert (c.category, c.retryable) == (category, retryable) assert c.retry_after == 3.0 and c.request_id == "req_x" assert 3.0 <= c.backoff(1) <= 3.3