Repository Public
1// Documentation content. Static, authored markup only (no user data is ever interpolated here).2// Dynamic parts are placeholders hydrated by docs.js:3// <div data-sample="id"> multi-language code tabs (samples.js)4// <div data-snip="n"> single code block registered with code()5// <div data-dyn="params|schema|errors|limits|endpoints|models|benchmarks|changelog"> rendered from live endpoints6// <span data-limit="key"> a value from openapi x-limits78export const SNIPPETS = [];9function code(lang, title, text) {10 SNIPPETS.push({ lang, title, text });11 return `<div data-snip="${SNIPPETS.length - 1}"></div>`;12}13const INFO = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="9"/><path d="M12 11v5"/><circle cx="12" cy="7.800" r=".6" fill="currentColor"/></svg>';14const note = (html, kind = "") => `<div class="callout ${kind}" role="note">${INFO}<div>${html}</div></div>`;15const r = String.raw;1617/** English descriptions merged into the tables generated from /openapi.json (name, type, constraints and default come from the backend). */18export const PARAM_DOCS = {19 ChatRequest: {20 model: "Model to run. Use an id returned by <code>GET /v1/models</code>, for example <code>maclustr-qwen3</code>.",21 messages: "The conversation so far, oldest first. See the <a href=\"#schema-Message\">Message</a> object.",22 stream: "When <code>true</code>, the response is a <a href=\"/docs/streaming\">server-sent event stream</a> instead of a single JSON object.",23 stream_options: "Streaming options. Set <code>include_usage</code> to receive a final usage chunk.",24 max_tokens: "Upper bound on generated tokens. The server clamps it to the model's <code>max_output_tokens</code> and to the room left in the context window. When omitted, the model's default applies.",25 max_completion_tokens: "Alias of <code>max_tokens</code>. Takes precedence when both are sent.",26 temperature: "Sampling temperature. Lower is more deterministic. When omitted, the engine default applies.",27 top_p: "Nucleus sampling: keep the smallest set of tokens whose cumulative probability reaches <code>top_p</code>.",28 top_k: "Sample only among the <code>top_k</code> most likely tokens. Not part of the OpenAI API.",29 min_p: "Discard tokens whose probability is below <code>min_p</code> times that of the most likely token. Not part of the OpenAI API.",30 presence_penalty: "Penalises tokens that already appeared, encouraging new topics.",31 frequency_penalty: "Penalises tokens in proportion to how often they already appeared.",32 repetition_penalty: "Multiplicative repetition penalty applied by the engine. Not part of the OpenAI API.",33 stop: "One string, or up to 8 strings (non-empty, at most 128 characters each), at which generation stops.",34 seed: "Seed for the sampler. The same seed and parameters make sampling repeatable on the same deployment.",35 n: "Number of choices. Only <code>1</code> is accepted.",36 user: "Accepted for OpenAI compatibility. Not used by the server.",37 tools: "Functions the model may call. See <a href=\"/docs/tool-calling\">Tool Calling</a>.",38 tool_choice: "Only <code>\"auto\"</code> and <code>\"none\"</code> are supported. Any other value, including forcing a specific function, is rejected with <code>400 unsupported_parameter</code>.",39 response_format: "<code>json_object</code> or <code>json_schema</code> to request JSON validated by the server. See <a href=\"/docs/structured-outputs\">Structured Outputs</a>.",40 enable_thinking: "Asks a reasoning-capable model to think before answering; the reasoning is returned as <code>reasoning_content</code>. Off by default, ignored by models without the <code>thinking</code> capability, and always off in JSON modes.",41 reasoning_effort: "OpenAI-style alternative to <code>enable_thinking</code>: any value other than <code>\"none\"</code> or <code>\"minimal\"</code> turns thinking on. <code>enable_thinking</code> wins when both are sent.",42 logprobs: "When <code>true</code>, token log-probabilities are requested from the engine and returned in <code>choices[].logprobs</code> on non-streaming responses.",43 top_logprobs: "Number of most likely alternatives to return per token. Requires <code>logprobs</code>.",44 },45 Message: {46 role: "<code>developer</code> is treated as <code>system</code>.",47 content: "Text of the message. An array of content parts is accepted if every part is text (<code>text</code>, <code>input_text</code> or <code>output_text</code>); the parts are concatenated. Images and other part types are rejected with <code>400</code>.",48 name: "Optional name. Forwarded for <code>tool</code> messages only.",49 tool_calls: "On an <code>assistant</code> message you replay: the tool calls the model made, exactly as received.",50 tool_call_id: "On a <code>tool</code> message: the <code>id</code> of the tool call this result answers.",51 },52 Tool: { type: "Always <code>function</code>.", function: "The function definition." },53 FunctionDef: {54 name: "Function name the model will use in its tool call.",55 description: "What the function does and when to call it. The model relies on this text.",56 parameters: "JSON Schema of the arguments object.",57 strict: "Accepted for OpenAI compatibility. The gateway does not validate tool-call arguments against your schema.",58 },59 ToolCall: { id: "Identifier to echo back in the matching <code>tool</code> message.", type: "Always <code>function</code>.", function: "Function name and arguments." },60 ToolCallFunction: { name: "Name of the called function.", arguments: "Arguments as a JSON-encoded string. Parse it, and validate it before use." },61 ResponseFormat: {62 type: "<code>text</code> (default behaviour), <code>json_object</code>, or <code>json_schema</code>.",63 json_schema: "Required when <code>type</code> is <code>json_schema</code>.",64 },65 JsonSchemaSpec: {66 name: "Accepted for OpenAI compatibility. Not used by the server.",67 description: "Accepted for OpenAI compatibility. Not used by the server.",68 schema: "The JSON Schema the output must satisfy. Checked as a Draft 2020-12 schema; an invalid schema is rejected with <code>400</code>.",69 strict: "Accepted for OpenAI compatibility. Validation is always applied on non-streaming requests, whatever this flag says.",70 },71 StreamOptions: { include_usage: "Emit a last chunk with an empty <code>choices</code> array and the <code>usage</code> object, just before <code>[DONE]</code>." },72};7374/** English text for the error codes listed in openapi x-error-codes (codes and HTTP statuses come from the backend). */75export const ERROR_DOCS = {76 invalid_request: ["The request body is invalid: a required field is missing, or a type or value is out of range. <code>param</code> names the field.", "No. Fix the request."],77 context_length_exceeded: ["The prompt does not fit in the model's context window.", "No. Shorten the conversation."],78 unsupported_parameter: ["A parameter that exists in the OpenAI API is not supported by this engine, for example a <code>tool_choice</code> other than <code>auto</code> or <code>none</code>.", "No. Remove the parameter."],79 missing_api_key: ["No API key was sent.", "No. Send <code>Authorization: Bearer …</code>."],80 invalid_api_key: ["The key is unknown, revoked, disabled or expired.", "No. Use a valid key."],81 model_not_allowed: ["The key is not allowed to use this model.", "No."],82 endpoint_not_allowed: ["The key is not allowed to use this endpoint.", "No."],83 model_not_found: ["The model does not exist.", "No. Check <code>GET /v1/models</code>."],84 request_too_large: ["The request body exceeds the maximum size.", "No. Send a smaller body."],85 json_validation_failed: ["JSON mode: the model output could not be validated, even after the automatic repair attempt.", "Possibly. A new attempt may succeed; simplifying the schema or the prompt helps more."],86 rate_limit_exceeded: ["The key reached its requests-per-minute limit.", "Yes, after <code>Retry-After</code>."],87 tokens_rate_limit_exceeded: ["The key reached its tokens-per-minute limit.", "Yes, after <code>Retry-After</code>."],88 concurrency_limit_exceeded: ["The key already has its maximum number of requests in flight.", "Yes, when one of your requests finishes."],89 insufficient_quota: ["The key's daily or monthly token quota is exhausted.", "Only after the quota resets. Do not retry in a loop."],90 too_many_attempts: ["Too many failed authentication attempts from this address.", "Yes, after <code>Retry-After</code>, with a valid key."],91 internal_error: ["Unexpected server error.", "Yes, with backoff. Report the <code>request_id</code> if it persists."],92 engine_error: ["The inference engine returned an error, or the connection to it failed or was lost (in which case the HTTP status can be <code>503</code>). Can also arrive as an in-stream error event.", "Yes, with backoff."],93 engine_overloaded: ["The waiting queue is full. The request was rejected immediately (backpressure).", "Yes, after <code>Retry-After</code>."],94 engine_unavailable: ["The inference engine is not available, typically while it restarts.", "Yes, after <code>Retry-After</code>."],95 memory_pressure: ["The server is under critical memory pressure and is shedding load.", "Yes, after <code>Retry-After</code>."],96 queue_timeout: ["The request waited in the queue longer than the queue timeout.", "Yes, after <code>Retry-After</code>."],97 engine_timeout: ["The engine did not answer in time. Can also arrive as an in-stream error event.", "Yes, with backoff."],98 request_timeout: ["Generation exceeded the maximum request duration. Can also arrive as an in-stream error event.", "Yes, with a smaller <code>max_tokens</code>."],99};100101export const LIMIT_DOCS = {102 max_body_bytes: ["Maximum request body", "bytes", "Larger bodies are rejected with <code>413 request_too_large</code>."],103 max_messages: ["Maximum messages per request", "messages", "More messages are rejected with <code>400</code>."],104 request_timeout_s: ["Maximum generation time", "seconds", "Beyond it the request ends with <code>504 request_timeout</code> (an in-stream error when streaming)."],105 queue_timeout_s: ["Maximum time in the queue", "seconds", "Beyond it the request ends with <code>503 queue_timeout</code>."],106 max_queue: ["Queue capacity", "requests", "When the queue is full, new requests get <code>503 engine_overloaded</code> immediately."],107};108109export const ENDPOINT_DOCS = {110 "GET /v1/models": "List the models your key can use.",111 "GET /v1/models/{model}": "Describe one model.",112 "POST /v1/chat/completions": "Create a chat completion, streamed or not.",113 "POST /v1/completions": "Create a raw text completion from a single prompt.",114 "GET /health": "Liveness of the gateway. Does not depend on the engine. No key required.",115 "GET /ready": "200 when an engine is available for every public model, 503 otherwise. No key required.",116};117118export const SECTIONS = [119 // =====================================================================================120 {121 slug: "overview", title: "Overview", group: "Introduction",122 lede: "MacLustr AI serves large language models from Apple Silicon hardware with MLX, behind an API that follows the OpenAI conventions.",123 html: () => r`124<p>If your code already talks to the OpenAI API, it can talk to MacLustr by changing two values: the base URL and the API key. Requests go to a gateway that authenticates the key, applies its limits, queues the request by priority, and hands it to an MLX inference engine running on the same machine.</p>125<div class="endpoint"><span class="badge accent">Base URL</span><span>https://llm.spboucher.ai/v1</span></div>126127<h2 id="endpoints">Endpoints</h2>128<p>This list is read from the live <a href="/openapi.json">OpenAPI contract</a> of this deployment.</p>129<div data-dyn="endpoints"></div>130131<h2 id="what-you-get">What the platform does</h2>132<ul>133 <li><strong>Chat completions</strong> with system, user, assistant and tool messages, sampling controls, stop sequences and seeds.</li>134 <li><strong>Streaming</strong> over server-sent events, with keepalive comments, an optional usage chunk and explicit in-stream errors.</li>135 <li><strong>Tool calling</strong> with <code>tool_choice</code> <code>auto</code> or <code>none</code>.</li>136 <li><strong>JSON output</strong> validated by the server against <code>json_object</code> or a JSON Schema.</li>137 <li><strong>Per-application keys</strong> with rate limits, token quotas, concurrency caps and priority classes.</li>138</ul>139140<h2 id="compatibility">Differences from the OpenAI API</h2>141<p>The request and response shapes are the same, but this is a different engine. These are the differences that matter in practice:</p>142<ul>143 <li><strong>Text only.</strong> Message content is text. Image, audio and file content parts are rejected with <code>400</code>.</li>144 <li><strong>One choice.</strong> <code>n</code> can only be <code>1</code>.</li>145 <li><strong>Tool choice.</strong> Only <code>auto</code> and <code>none</code>. Forcing a function is not supported.</li>146 <li><strong>JSON modes are validated, not grammar-constrained.</strong> Validity is guaranteed for non-streaming requests only. See <a href="/docs/structured-outputs">Structured Outputs</a>.</li>147 <li><strong>Tool calls are not fragmented.</strong> When streaming, each tool call arrives whole in one chunk.</li>148 <li><strong>Extra fields.</strong> <code>reasoning_content</code> on messages and deltas, <code>top_k</code>, <code>min_p</code>, <code>repetition_penalty</code> and <code>enable_thinking</code> on requests.</li>149 <li><strong>Unknown fields are ignored</strong> rather than rejected, so SDKs that send extra parameters keep working.</li>150 <li><strong>Other OpenAI endpoints</strong> (embeddings, images, audio, files, assistants, responses) are not implemented.</li>151</ul>152153<h2 id="privacy">Local inference and privacy</h2>154<p>Inference runs on our own Apple Silicon hardware; no third-party model API is called. The gateway does not write prompts or completions to its logs or its database: it records counters and timings only (token counts, latency, status, the key used). API keys are stored hashed.</p>155156<h2 id="next">Where to go next</h2>157<div class="cards">158 <a class="card-link" href="/docs/getting-started"><strong>Getting Started</strong><span>Your first request in five minutes.</span></a>159 <a class="card-link" href="/docs/chat-completions"><strong>Chat Completions</strong><span>Every parameter, generated from the backend schema.</span></a>160 <a class="card-link" href="/docs/sdk-examples"><strong>SDK Examples</strong><span>Python, TypeScript, Swift and cURL clients.</span></a>161 <a class="card-link" href="/playground"><strong>Playground</strong><span>Try the model against the real API.</span></a>162</div>`,163 },164165 // =====================================================================================166 {167 slug: "getting-started", title: "Getting Started", group: "Introduction",168 lede: "Make a first request, read the response, then stream one.",169 html: () => r`170<h2 id="get-a-key">1. Get an API key</h2>171<p>Keys are issued by the administrator of this MacLustr instance; there is no self-service sign-up. A key looks like <code>mlx_sk_…</code> and is displayed only once, when it is created. Store it in a secret manager or an environment variable.</p>172${code("bash", "Shell", `export MACLUSTR_API_KEY="YOUR_MACLUSTR_API_KEY"`)}173174<h2 id="first-request">2. Send a request</h2>175<p>The examples use the official OpenAI SDKs for Python and TypeScript, the native <code>URLSession</code> for Swift, and cURL. Pick a language once; the choice is remembered across the site.</p>176<div data-sample="chat" data-label="First chat completion"></div>177178<h2 id="read-the-response">3. Read the response</h2>179<p>The answer is in <code>choices[0].message.content</code>. <code>usage</code> reports the tokens consumed, including how many prompt tokens were served from the prefix cache.</p>180${code("json", "Example response", `{181 "id": "chatcmpl-8c4fc9edc7e1df4ff3dd16f0",182 "object": "chat.completion",183 "created": 1795098973,184 "model": "maclustr-qwen3",185 "system_fingerprint": "fp_maclustr_mlx",186 "choices": [187 {188 "index": 0,189 "message": {190 "role": "assistant",191 "content": "Unified memory lets the CPU and GPU share one pool of RAM, so model weights are never copied between them."192 },193 "finish_reason": "stop",194 "logprobs": null195 }196 ],197 "usage": {198 "prompt_tokens": 27,199 "completion_tokens": 24,200 "total_tokens": 51,201 "prompt_tokens_details": { "cached_tokens": 0 }202 }203}`)}204205<h2 id="stream-it">4. Stream the answer</h2>206<p>Set <code>stream</code> to <code>true</code> to receive tokens as they are generated. Add <code>stream_options.include_usage</code> to get the token counts at the end.</p>207<div data-sample="stream" data-label="Streaming chat completion"></div>208209<h2 id="existing-code">5. Point existing code at MacLustr</h2>210<p>Code written for the OpenAI SDKs needs two changes:</p>211<ol>212 <li>Set the base URL to <code>https://llm.spboucher.ai/v1</code>.</li>213 <li>Use your MacLustr key and a MacLustr model id such as <code>maclustr-qwen3</code>.</li>214</ol>215<p>Then check the <a href="/docs/overview#compatibility">differences from the OpenAI API</a> for the features you rely on.</p>216<div class="cards">217 <a class="card-link" href="/docs/authentication"><strong>Authentication</strong><span>How keys work and how to keep them safe.</span></a>218 <a class="card-link" href="/docs/error-handling"><strong>Error Handling</strong><span>The error envelope and what to retry.</span></a>219 <a class="card-link" href="/docs/rate-limits"><strong>Rate Limits</strong><span>Limits, quotas, queueing and priorities.</span></a>220</div>`,221 },222223 // =====================================================================================224 {225 slug: "authentication", title: "Authentication", group: "Introduction",226 lede: "Every request to /v1 is authenticated with a bearer API key.",227 html: () => r`228<h2 id="bearer">Bearer keys</h2>229<p>Send the key in the <code>Authorization</code> header. This includes <code>GET /v1/models</code>: no <code>/v1</code> endpoint is public.</p>230${code("http", "HTTP", `POST /v1/chat/completions HTTP/1.1231Host: llm.spboucher.ai232Authorization: Bearer YOUR_MACLUSTR_API_KEY233Content-Type: application/json`)}234<p>Keys start with <code>mlx_sk_</code>. A well-formed request without a valid key returns <code>401</code>. A request whose body fails validation (a missing <code>model</code>, for example) can be answered with <code>400</code> first, so do not use a malformed request to test a key: call <code>GET /v1/models</code> instead.</p>235236<h2 id="per-app">One key per application</h2>237<p>Create a separate key for each application, service or environment. Every key carries its own configuration:</p>238<ul>239 <li><strong>Scope:</strong> the models and the endpoints (<code>chat.completions</code>, <code>completions</code>, <code>models</code>) it may use.</li>240 <li><strong>Limits:</strong> requests per minute, tokens per minute, daily and monthly token quotas, and a concurrency cap. See <a href="/docs/rate-limits">Rate Limits</a>.</li>241 <li><strong>Priority class:</strong> <code>admin</code>, <code>internal</code>, <code>client</code> or <code>background</code>, used by the scheduler when requests queue.</li>242 <li><strong>Lifecycle:</strong> an optional expiry date; a key can be disabled, rotated (a new secret replaces the old one) or revoked at any time.</li>243</ul>244<p>Separate keys keep usage attributable, and let you revoke one application without touching the others.</p>245246<h2 id="storage">Keys are stored hashed and shown once</h2>247<p>The server stores a keyed hash of each key, never the key itself. The full secret is displayed a single time, at creation or rotation. If it is lost, it cannot be recovered: rotate the key to obtain a new secret.</p>248249<h2 id="safety">Keep keys out of client-side code</h2>250${note(`<p><strong>Never ship a key in a browser bundle, a mobile app binary or a public repository.</strong> Anything delivered to a user's device can be extracted.</p>`, "warn")}251<ul>252 <li>Call MacLustr from your own backend, and let your backend authenticate your users.</li>253 <li>Load the key from an environment variable or a secret manager, not from source code.</li>254 <li>For a native app that must call the API directly, give it its own narrowly scoped key with low limits, delivered at runtime by your backend and stored in the Keychain. Treat it as extractable, and rotate it.</li>255 <li>The <code>/v1</code> endpoints answer cross-origin requests (CORS is open, without credentials) because bearer tokens are not ambient credentials. That makes browser calls technically possible; it does not make embedding a key in a web page safe.</li>256</ul>257258<h2 id="auth-errors">Authentication errors</h2>259<div class="table-wrap"><table class="data">260<thead><tr><th>Status</th><th>Code</th><th>Meaning</th></tr></thead>261<tbody>262<tr><td class="num">401</td><td><code>missing_api_key</code></td><td>No <code>Authorization: Bearer</code> header.</td></tr>263<tr><td class="num">401</td><td><code>invalid_api_key</code></td><td>Unknown, revoked, disabled or expired key.</td></tr>264<tr><td class="num">403</td><td><code>model_not_allowed</code></td><td>The key is valid but not scoped to this model.</td></tr>265<tr><td class="num">403</td><td><code>endpoint_not_allowed</code></td><td>The key is valid but not scoped to this endpoint.</td></tr>266<tr><td class="num">429</td><td><code>too_many_attempts</code></td><td>Repeated invalid keys from the same address are throttled. Wait for <code>Retry-After</code>.</td></tr>267</tbody></table></div>268<p><code>401</code> responses carry a <code>WWW-Authenticate: Bearer</code> header. The body follows the standard <a href="/docs/error-handling">error envelope</a>.</p>`,269 },270271 // =====================================================================================272 {273 slug: "models", title: "Models", group: "API",274 lede: "The models served by this deployment, read live from the public status endpoint.",275 html: () => r`276<h2 id="available">Available now</h2>277<div data-dyn="models"></div>278<p class="source-note">Source: <code>GET /api/status</code>, fetched by your browser when this page loaded.</p>279280<h2 id="choosing">Using a model</h2>281<p>Pass the model <code>id</code> in the <code>model</code> field of your request. An id that does not exist returns <code>404 model_not_found</code>; an id your key is not scoped to returns <code>403 model_not_allowed</code>.</p>282<p>The <strong>context window</strong> is the maximum number of tokens the prompt and the completion can occupy together. A prompt that leaves no room for an answer is rejected with <code>400 context_length_exceeded</code>.</p>283284<h2 id="list">List models with the API</h2>285<p><code>GET /v1/models</code> requires an API key and returns only the models that key can use. <code>GET /v1/models/{model}</code> returns a single entry.</p>286<div data-sample="models" data-label="List models"></div>287<h3 id="model-object">The model object</h3>288<div class="table-wrap"><table class="data">289<thead><tr><th>Field</th><th>Type</th><th>Description</th></tr></thead>290<tbody>291<tr><td class="param-name">id</td><td class="param-type">string</td><td>Identifier to pass as <code>model</code>.</td></tr>292<tr><td class="param-name">object</td><td class="param-type">"model"</td><td>Object type.</td></tr>293<tr><td class="param-name">owned_by</td><td class="param-type">string</td><td>Owner label.</td></tr>294<tr><td class="param-name">display_name</td><td class="param-type">string</td><td>Human-readable name.</td></tr>295<tr><td class="param-name">context_window</td><td class="param-type">integer</td><td>Maximum prompt plus completion tokens.</td></tr>296<tr><td class="param-name">max_output_tokens</td><td class="param-type">integer</td><td>Hard cap on <code>max_tokens</code> for one response.</td></tr>297<tr><td class="param-name">quantization</td><td class="param-type">string</td><td>Weight format served by the engine.</td></tr>298<tr><td class="param-name">license</td><td class="param-type">string</td><td>License of the model weights.</td></tr>299<tr><td class="param-name">source</td><td class="param-type">object</td><td><code>repo</code> and <code>revision</code> of the weights.</td></tr>300<tr><td class="param-name">capabilities</td><td class="param-type">object</td><td><code>chat</code>, <code>streaming</code>, <code>tools</code>, <code>thinking</code> (booleans) and <code>json_mode</code> (<code>"validated"</code>).</td></tr>301<tr><td class="param-name">description</td><td class="param-type">string</td><td>Short description.</td></tr>302</tbody></table></div>`,303 },304305 // =====================================================================================306 {307 slug: "chat-completions", title: "Chat Completions", group: "API",308 lede: "Generate the next assistant message of a conversation.",309 html: () => r`310<div class="endpoint"><span class="method post">POST</span><span>https://llm.spboucher.ai/v1/chat/completions</span></div>311312<h2 id="request">Request</h2>313<div data-sample="chat" data-label="Chat completion"></div>314${code("json", "Request body", `{315 "model": "maclustr-qwen3",316 "messages": [317 { "role": "system", "content": "You are a concise assistant." },318 { "role": "user", "content": "Explain unified memory in one sentence." }319 ],320 "temperature": 0.7,321 "top_p": 0.9,322 "max_tokens": 256,323 "stop": ["\\n\\n"],324 "seed": 42325}`)}326327<h2 id="parameters">Request parameters</h2>328<p>This table is generated in your browser from <code>components.schemas.ChatRequest</code> in the live <a href="/openapi.json">OpenAPI contract</a>: names, types, constraints and defaults are the ones the backend enforces. A default of “—” means the field is optional and the model or engine default applies.</p>329<div data-dyn="params" data-schema="ChatRequest"></div>330331<h3 id="schema-Message">Message</h3>332<div data-dyn="params" data-schema="Message"></div>333<h3 id="schema-StreamOptions">StreamOptions</h3>334<div data-dyn="params" data-schema="StreamOptions"></div>335<h3 id="schema-Tool">Tool</h3>336<div data-dyn="params" data-schema="Tool"></div>337<h3 id="schema-FunctionDef">FunctionDef</h3>338<div data-dyn="params" data-schema="FunctionDef"></div>339<h3 id="schema-ToolCall">ToolCall</h3>340<div data-dyn="params" data-schema="ToolCall"></div>341<h3 id="schema-ToolCallFunction">ToolCallFunction</h3>342<div data-dyn="params" data-schema="ToolCallFunction"></div>343<h3 id="schema-ResponseFormat">ResponseFormat</h3>344<div data-dyn="params" data-schema="ResponseFormat"></div>345<h3 id="schema-JsonSchemaSpec">JsonSchemaSpec</h3>346<div data-dyn="params" data-schema="JsonSchemaSpec"></div>347348<h2 id="behaviour">Behaviour worth knowing</h2>349<ul>350 <li><strong>Unknown fields are ignored.</strong> A typo in a parameter name is silently dropped, not rejected.</li>351 <li><strong>Output length.</strong> The effective limit is the smallest of your <code>max_tokens</code>, the model's <code>max_output_tokens</code>, and the room left in the context window after the prompt. If generation stops there, <code>finish_reason</code> is <code>length</code>.</li>352 <li><strong>Context overflow.</strong> If the prompt leaves fewer than 16 tokens of room, the request is rejected with <code>400 context_length_exceeded</code> before anything is generated.</li>353 <li><strong>Reasoning is opt-in.</strong> Models that can think do so only when <code>enable_thinking</code> or <code>reasoning_effort</code> asks for it.</li>354 <li><strong>Validation errors</strong> return <code>400 invalid_request</code>, with <code>param</code> set to the offending field.</li>355</ul>356357<h2 id="response">Response</h2>358${code("json", "200 OK · application/json", `{359 "id": "chatcmpl-8c4fc9edc7e1df4ff3dd16f0",360 "object": "chat.completion",361 "created": 1795098973,362 "model": "maclustr-qwen3",363 "system_fingerprint": "fp_maclustr_mlx",364 "choices": [365 {366 "index": 0,367 "message": {368 "role": "assistant",369 "content": "Unified memory lets the CPU and GPU share one pool of RAM, so model weights are never copied between them."370 },371 "finish_reason": "stop",372 "logprobs": null373 }374 ],375 "usage": {376 "prompt_tokens": 27,377 "completion_tokens": 24,378 "total_tokens": 51,379 "prompt_tokens_details": { "cached_tokens": 0 }380 }381}`)}382<div class="table-wrap"><table class="data">383<thead><tr><th>Field</th><th>Type</th><th>Description</th></tr></thead>384<tbody>385<tr><td class="param-name">id</td><td class="param-type">string</td><td><code>chatcmpl-</code> followed by the request id (also sent as the <code>x-request-id</code> header).</td></tr>386<tr><td class="param-name">object</td><td class="param-type">"chat.completion"</td><td>Object type.</td></tr>387<tr><td class="param-name">created</td><td class="param-type">integer</td><td>Unix time, in seconds.</td></tr>388<tr><td class="param-name">model</td><td class="param-type">string</td><td>The model that answered.</td></tr>389<tr><td class="param-name">system_fingerprint</td><td class="param-type">string</td><td>Constant identifier of the serving stack.</td></tr>390<tr><td class="param-name">choices[0].message.role</td><td class="param-type">"assistant"</td><td></td></tr>391<tr><td class="param-name">choices[0].message.content</td><td class="param-type">string | null</td><td>The answer. <code>null</code> when the message only contains tool calls.</td></tr>392<tr><td class="param-name">choices[0].message.reasoning_content</td><td class="param-type">string</td><td>Present only when the model produced reasoning.</td></tr>393<tr><td class="param-name">choices[0].message.tool_calls</td><td class="param-type">ToolCall[]</td><td>Present only when the model called tools.</td></tr>394<tr><td class="param-name">choices[0].finish_reason</td><td class="param-type">string</td><td><code>stop</code> (natural end or stop sequence), <code>length</code> (token limit reached) or <code>tool_calls</code>.</td></tr>395<tr><td class="param-name">choices[0].logprobs</td><td class="param-type">object | null</td><td><code>{ "content": [...] }</code> when <code>logprobs</code> was requested, otherwise <code>null</code>.</td></tr>396<tr><td class="param-name">usage.prompt_tokens</td><td class="param-type">integer</td><td>Tokens in the prompt, after the chat template is applied.</td></tr>397<tr><td class="param-name">usage.completion_tokens</td><td class="param-type">integer</td><td>Tokens generated.</td></tr>398<tr><td class="param-name">usage.total_tokens</td><td class="param-type">integer</td><td>Sum of the two.</td></tr>399<tr><td class="param-name">usage.prompt_tokens_details.cached_tokens</td><td class="param-type">integer</td><td>Prompt tokens served from the KV prefix cache.</td></tr>400</tbody></table></div>401402<h2 id="headers">Response headers</h2>403<div class="table-wrap"><table class="data">404<thead><tr><th>Header</th><th>Description</th></tr></thead>405<tbody>406<tr><td class="param-name">x-request-id</td><td>Unique id of the request. Quote it when reporting a problem.</td></tr>407<tr><td class="param-name">x-ratelimit-limit-requests</td><td>The key's requests-per-minute limit. Present only if the key has one.</td></tr>408<tr><td class="param-name">x-ratelimit-remaining-requests</td><td>Requests left in the current window.</td></tr>409<tr><td class="param-name">x-ratelimit-limit-tokens</td><td>The key's tokens-per-minute limit. Present only if the key has one.</td></tr>410<tr><td class="param-name">x-ratelimit-remaining-tokens</td><td>Tokens left in the current window.</td></tr>411<tr><td class="param-name">Retry-After</td><td>On <code>429</code> and <code>503</code>: seconds to wait before retrying.</td></tr>412</tbody></table></div>413414<h2 id="text-completions">Raw text completions</h2>415<p><code>POST /v1/completions</code> takes a single <code>prompt</code> string (or an array containing exactly one string) and the same sampling parameters, and returns a <code>text_completion</code> object with <code>choices[0].text</code>. No chat template is applied. It supports streaming in the same way.</p>`,416 },417418 // =====================================================================================419 {420 slug: "streaming", title: "Streaming", group: "API",421 lede: "Receive tokens as they are generated, over server-sent events.",422 html: () => r`423<h2 id="enable">Enable streaming</h2>424<p>Send <code>"stream": true</code>. The response has <code>Content-Type: text/event-stream</code> and stays open until generation ends. Add <code>"stream_options": {"include_usage": true}</code> to receive token counts in a final chunk.</p>425<div data-sample="stream" data-label="Streaming chat completion"></div>426427<h2 id="anatomy">Anatomy of a stream</h2>428<p>Each event is one <code>data:</code> line followed by a blank line. This is a complete stream, shortened for readability (every chunk also carries <code>id</code>, <code>object</code>, <code>created</code>, <code>model</code> and <code>system_fingerprint</code>):</p>429${code("sse", "text/event-stream", `data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}430431: keepalive432433data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning_content":"The user wants one sentence."},"finish_reason":null}]}434435data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Unified"},"finish_reason":null}]}436437data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":" memory"},"finish_reason":null}]}438439data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}440441data: {"object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":27,"completion_tokens":24,"total_tokens":51,"prompt_tokens_details":{"cached_tokens":0}}}442443data: [DONE]`)}444<ol>445 <li><strong>Role chunk.</strong> The first event always has <code>delta: {"role": "assistant", "content": ""}</code>. It is sent as soon as the engine accepts the request.</li>446 <li><strong>Keepalive comments.</strong> Lines starting with a colon, such as <code>: keepalive</code>, are SSE comments. The server sends them while the prompt is being processed, so proxies and clients do not time out before the first token. Ignore them; the OpenAI SDKs already do.</li>447 <li><strong>Reasoning deltas.</strong> If the model thinks, <code>delta.reasoning_content</code> carries the reasoning text, before the answer. It never appears unless you asked for thinking.</li>448 <li><strong>Content deltas.</strong> <code>delta.content</code> carries the answer, piece by piece. Concatenate them.</li>449 <li><strong>Tool calls.</strong> <code>delta.tool_calls</code> carries complete calls. See <a href="/docs/tool-calling#streaming">Tool Calling</a>.</li>450 <li><strong>Finish chunk.</strong> An empty <code>delta</code> with <code>finish_reason</code> set to <code>stop</code>, <code>length</code> or <code>tool_calls</code>.</li>451 <li><strong>Usage chunk.</strong> Only with <code>include_usage</code>: <code>choices</code> is an empty array and <code>usage</code> holds the token counts. Guard against the empty array before reading <code>choices[0]</code>.</li>452 <li><strong>Terminator.</strong> The literal line <code>data: [DONE]</code>, then the connection closes.</li>453</ol>454455<h2 id="errors">Errors during a stream</h2>456<p>Errors detected before generation starts (authentication, validation, rate limits, a full queue) are returned as normal HTTP errors with a JSON body. Once the stream has started, the HTTP status is already <code>200</code>, so a failure is reported as an event whose payload is the standard error envelope. After it, the stream ends <strong>without</strong> a finish chunk and without <code>[DONE]</code>:</p>457${code("sse", "In-stream error", `data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Unified"},"finish_reason":null}]}458459data: {"error":{"message":"…","type":"api_error","param":null,"code":"engine_error","request_id":"750f45ffa6457aeb5fcfe50c"}}`)}460<p>Always check each event for an <code>error</code> key, and treat a stream that closes before <code>[DONE]</code> as incomplete. The codes you can receive this way are <code>engine_error</code> (the connection to the engine was lost), <code>engine_timeout</code> (the engine stopped answering) and <code>request_timeout</code> (generation ran longer than <span data-limit="request_timeout_s">the maximum request duration</span>).</p>461462<h2 id="cancel">Cancelling</h2>463<p>To stop a generation, close the connection: abort the <code>fetch</code>, cancel the Swift <code>Task</code>, or break out of the SDK iterator and close the stream. The server detects the disconnect and stops generating, which frees your concurrency slot. Tokens generated before the cancellation still count towards your usage.</p>464465<h2 id="reconnect">Reconnection</h2>466<p>A stream cannot be resumed. There are no event ids and <code>Last-Event-ID</code> is not supported. If the connection drops, keep what you received if it is useful, then send the request again, with the <a href="/docs/error-handling#retries">same backoff rules</a> as any other retry. Do not use the browser <code>EventSource</code> API: it cannot send a <code>POST</code> body or an <code>Authorization</code> header, and it reconnects on its own.</p>467468<h2 id="proxies">Buffering proxies</h2>469<p>The stream is sent with <code>Cache-Control: no-cache, no-transform</code> and <code>X-Accel-Buffering: no</code>. If you relay it through your own backend or reverse proxy, disable response buffering and compression for this route, or tokens will arrive in bursts.</p>`,470 },471472 // =====================================================================================473 {474 slug: "tool-calling", title: "Tool Calling", group: "Capabilities",475 lede: "Let the model ask your code to run functions, then continue with the results.",476 html: () => r`477<h2 id="how">How it works</h2>478<ol class="steps">479 <li>You send the conversation together with a list of <code>tools</code>, each described by a name, a description and a JSON Schema for its arguments.</li>480 <li>If the model decides to use one, the response has <code>finish_reason: "tool_calls"</code> and <code>message.tool_calls</code> instead of text.</li>481 <li>Your code runs the function. The model never executes anything itself.</li>482 <li>You append the assistant message (with its <code>tool_calls</code>) and one <code>tool</code> message per call, carrying the result and the matching <code>tool_call_id</code>, then call the API again.</li>483 <li>The model answers in text, or asks for more calls. Loop until <code>finish_reason</code> is <code>stop</code>.</li>484</ol>485486<h2 id="round-trip">Full round trip</h2>487<div data-sample="tools" data-label="Tool calling round trip"></div>488489<h2 id="response">What the model returns</h2>490${code("json", "Response with a tool call", `{491 "id": "chatcmpl-dbde4211b9628ca3ca7148f6",492 "object": "chat.completion",493 "model": "maclustr-qwen3",494 "choices": [495 {496 "index": 0,497 "message": {498 "role": "assistant",499 "content": null,500 "tool_calls": [501 {502 "id": "call_237ed1f628bd498da864c4b7",503 "type": "function",504 "function": { "name": "get_weather", "arguments": "{\\"city\\": \\"Montreal\\"}" }505 }506 ]507 },508 "finish_reason": "tool_calls",509 "logprobs": null510 }511 ],512 "usage": { "prompt_tokens": 182, "completion_tokens": 21, "total_tokens": 203, "prompt_tokens_details": { "cached_tokens": 0 } }513}`)}514<p><code>function.arguments</code> is a <strong>JSON-encoded string</strong>, not an object. Parse it, and validate it against your own schema before acting on it: the model can produce arguments that are well-formed but wrong.</p>515516<h2 id="results">Sending results back</h2>517${code("json", "Messages of the second request", `[518 { "role": "user", "content": "What is the weather in Montreal?" },519 {520 "role": "assistant",521 "content": null,522 "tool_calls": [523 {524 "id": "call_237ed1f628bd498da864c4b7",525 "type": "function",526 "function": { "name": "get_weather", "arguments": "{\\"city\\": \\"Montreal\\"}" }527 }528 ]529 },530 {531 "role": "tool",532 "tool_call_id": "call_237ed1f628bd498da864c4b7",533 "content": "{\\"temperature_c\\": -4, \\"conditions\\": \\"light snow\\"}"534 }535]`)}536<p>The <code>content</code> of a tool message is a string; JSON is a good choice. Keep sending the same <code>tools</code> list on follow-up requests so the model can call again. When you replay an assistant message, its <code>function.arguments</code> must still be valid JSON, otherwise the request is rejected with <code>400</code>.</p>537538<h2 id="streaming">Tool calls when streaming</h2>539<p>Each tool call arrives <strong>whole, in a single chunk</strong>, with its <code>index</code>, <code>id</code>, <code>name</code> and complete <code>arguments</code>. You do not need to accumulate argument fragments as with the OpenAI API, although code that does accumulate keeps working.</p>540${code("sse", "text/event-stream", `data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}541542data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_870d56bbcb2e4ce4be7d3431","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\": \\"Montreal\\"}"}}]},"finish_reason":null}]}543544data: {"object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}545546data: [DONE]`)}547548<h2 id="limits">Limits, stated plainly</h2>549<ul>550 <li><strong><code>tool_choice</code> supports only <code>"auto"</code> and <code>"none"</code>.</strong> <code>"required"</code> and forcing a named function return <code>400 unsupported_parameter</code>. With <code>"none"</code>, the tools are not shown to the model at all.</li>551 <li><strong>Calls are not fragmented</strong> in streams: one chunk per call.</li>552 <li><strong>Arguments are not validated by the server.</strong> The <code>strict</code> flag is accepted and has no effect on validation.</li>553 <li><strong>Tools cannot be combined with JSON modes.</strong> Sending both <code>tools</code> and a JSON <code>response_format</code> returns <code>400</code>.</li>554 <li>The number of tools and the allowed characters in function names are listed in the <a href="/docs/chat-completions#schema-FunctionDef">generated schema tables</a>.</li>555 <li>Whether a model supports tools is reported by <code>capabilities.tools</code> in <code>GET /v1/models</code>.</li>556</ul>`,557 },558559 // =====================================================================================560 {561 slug: "structured-outputs", title: "Structured Outputs", group: "Capabilities",562 lede: "Ask for JSON, and let the server check that you got JSON.",563 html: () => r`564${note(`<p><strong>Read this first.</strong> JSON output on MacLustr is <strong>instruction-guided and validated server-side</strong>. It is <strong>not grammar-constrained</strong>: the sampler is free to produce any token. Validity is <strong>guaranteed only for non-streaming requests</strong>, where an invalid answer is never returned to you as a success.</p>`, "warn")}565566<h2 id="modes">Two modes</h2>567<div class="table-wrap"><table class="data">568<thead><tr><th>response_format.type</th><th>What is checked</th></tr></thead>569<tbody>570<tr><td class="param-name">json_object</td><td>The output parses as JSON and is an object or an array.</td></tr>571<tr><td class="param-name">json_schema</td><td>The same, and the value satisfies the JSON Schema you provide in <code>json_schema.schema</code>.</td></tr>572</tbody></table></div>573<div data-sample="json" data-label="Structured output with a JSON Schema"></div>574575<h2 id="how">What the server does</h2>576<ol class="steps">577 <li><strong>Instructs.</strong> An instruction to answer with a single JSON value, plus your schema in <code>json_schema</code> mode, is appended to your system message (or added as one). It counts towards <code>prompt_tokens</code>.</li>578 <li><strong>Generates</strong> the answer normally.</li>579 <li><strong>Validates.</strong> The output is parsed (a surrounding Markdown code fence is tolerated and removed) and, in <code>json_schema</code> mode, checked against your schema.</li>580 <li><strong>Repairs once.</strong> If validation fails, the server sends the model its own answer and the validation error, and asks for a corrected version. This happens at most one time.</li>581 <li><strong>Answers.</strong> On success, <code>message.content</code> is the validated JSON text. If the second attempt is also invalid, the request fails with <code>422 json_validation_failed</code>.</li>582</ol>583${code("json", "422 Unprocessable Entity", `{584 "error": {585 "message": "…",586 "type": "api_error",587 "param": null,588 "code": "json_validation_failed",589 "request_id": "9d45197b09710842c4a1c5ef"590 }591}`)}592593<h2 id="guarantees">Guarantees and their limits</h2>594<ul>595 <li><strong>Non-streaming:</strong> a <code>200</code> response with <code>finish_reason: "stop"</code> contains JSON that passed validation.</li>596 <li><strong>Truncation is not validated.</strong> If generation hits the token limit, the response is returned as is with <code>finish_reason: "length"</code>, and the JSON is probably incomplete. Check <code>finish_reason</code>, and set <code>max_tokens</code> high enough for your schema.</li>597 <li><strong>Streaming:</strong> the instruction is still added, but the output is forwarded as it is generated and is <strong>neither validated nor repaired</strong>. Parse it defensively, or do not stream when you need the guarantee.</li>598 <li><strong>Usage includes the repair.</strong> When a repair attempt happens, <code>usage</code> is the sum of both attempts, and the response takes about twice as long.</li>599 <li><strong>No tools.</strong> A JSON <code>response_format</code> cannot be combined with <code>tools</code> (<code>400</code>).</li>600 <li><strong>No reasoning.</strong> Thinking is turned off in JSON modes.</li>601 <li><strong>Schema errors are yours.</strong> A schema that is not a valid Draft 2020-12 JSON Schema is rejected with <code>400</code> before generation. <code>json_schema.name</code>, <code>description</code> and <code>strict</code> are accepted and not used.</li>602</ul>603604<h2 id="tips">Getting reliable results</h2>605<ul>606 <li>Describe the fields in your prompt as well as in the schema; the model follows instructions, not a grammar.</li>607 <li>Prefer flat schemas with explicit <code>required</code> lists and <code>additionalProperties: false</code>.</li>608 <li>Use a low <code>temperature</code>.</li>609 <li>Handle <code>422</code>: retry once, then fall back or surface the failure.</li>610</ul>`,611 },612613 // =====================================================================================614 {615 slug: "sdk-examples", title: "SDK Examples", group: "Capabilities",616 lede: "Complete, copyable clients for Python, TypeScript, Swift and cURL.",617 html: () => r`618<p>MacLustr does not publish its own SDK. The official OpenAI SDKs work once the base URL is changed; for Swift, a small native client is enough.</p>619620<h2 id="python">Python</h2>621<p>Uses the official <code>openai</code> package. <code>reasoning_content</code> is a MacLustr extension, so it is read with <code>getattr</code>.</p>622<div data-sdk="python" data-lang="python" data-title="Python · openai"></div>623<h3 id="python-async">Async</h3>624<div data-sdk="async_python" data-lang="python" data-title="Python · AsyncOpenAI"></div>625626<h2 id="typescript">JavaScript and TypeScript</h2>627<p>Uses the official <code>openai</code> package on Node.js, Bun or Deno. Run it on a server: a key embedded in a web page is public.</p>628<div data-sdk="typescript" data-lang="typescript" data-title="TypeScript · openai"></div>629630<h2 id="swift">Swift</h2>631<p>A dependency-free client built on <code>URLSession</code>. It covers bearer authentication, JSON requests, server-sent events with <code>URLSession.bytes</code>, the error envelope including in-stream errors, and cancellation through structured concurrency.</p>632<div data-sdk="swift" data-lang="swift" data-title="Swift · URLSession"></div>633<h3 id="swift-notes">Notes for Apple platforms</h3>634<ul>635 <li><strong>Cancellation.</strong> Cancelling the <code>Task</code> that consumes the stream terminates the <code>AsyncThrowingStream</code>, which cancels the underlying <code>URLSession</code> task and closes the connection. The server stops generating when it sees the disconnect. In SwiftUI, starting the stream from <code>.task { }</code> gives you this for free when the view disappears.</li>636 <li><strong>Reconnection.</strong> Streams cannot be resumed. On <code>URLError.networkConnectionLost</code> or <code>.timedOut</code> (common when an iPhone changes network or the app is suspended), keep the text already displayed and send the request again, with exponential backoff and a retry cap. Do not retry automatically after the user cancelled.</li>637 <li><strong>Background.</strong> iOS suspends network activity shortly after the app leaves the foreground. A long generation will be interrupted; plan for it rather than relying on background sessions, which do not support streaming reads.</li>638 <li><strong>Keys.</strong> Do not embed a key in the app binary. Fetch a scoped key from your backend after the user signs in, and store it in the Keychain.</li>639</ul>640641<h2 id="curl">cURL</h2>642<div data-sdk="curl" data-lang="bash" data-title="cURL"></div>`,643 },644645 // =====================================================================================646 {647 slug: "error-handling", title: "Error Handling", group: "Operations",648 lede: "One error envelope, stable codes, and clear rules for what to retry.",649 html: () => r`650<h2 id="envelope">The error envelope</h2>651<p>Every error response, on every endpoint, has the same JSON body:</p>652${code("json", "Error response", `{653 "error": {654 "message": "temperature: Input should be less than or equal to 2",655 "type": "invalid_request_error",656 "param": "temperature",657 "code": "invalid_request",658 "request_id": "23a4b2f9535f0eaacc121883"659 }660}`)}661<div class="table-wrap"><table class="data">662<thead><tr><th>Field</th><th>Description</th></tr></thead>663<tbody>664<tr><td class="param-name">message</td><td>Human-readable explanation. Server messages may be written in French; do not parse them. Branch on <code>code</code>.</td></tr>665<tr><td class="param-name">type</td><td>Broad category, see below.</td></tr>666<tr><td class="param-name">param</td><td>The request field at fault, or <code>null</code>.</td></tr>667<tr><td class="param-name">code</td><td>Stable machine-readable code. This is what your code should test.</td></tr>668<tr><td class="param-name">request_id</td><td>Same value as the <code>x-request-id</code> response header.</td></tr>669</tbody></table></div>670671<h2 id="types">Error types</h2>672<div class="table-wrap"><table class="data">673<thead><tr><th>type</th><th>Typical status</th><th>Meaning</th></tr></thead>674<tbody>675<tr><td class="param-name">invalid_request_error</td><td class="num">400, 413</td><td>The request is malformed or uses an unsupported option.</td></tr>676<tr><td class="param-name">authentication_error</td><td class="num">401</td><td>Missing or invalid API key.</td></tr>677<tr><td class="param-name">permission_error</td><td class="num">403</td><td>The key is valid but not allowed to do this.</td></tr>678<tr><td class="param-name">not_found_error</td><td class="num">404</td><td>Unknown model or route.</td></tr>679<tr><td class="param-name">rate_limit_error</td><td class="num">429</td><td>A limit or quota of <em>your key</em> was reached.</td></tr>680<tr><td class="param-name">overloaded_error</td><td class="num">503</td><td>The <em>server</em> cannot take the request right now.</td></tr>681<tr><td class="param-name">api_error</td><td class="num">422, 500, 502, 504</td><td>The server or the engine failed, or JSON validation failed.</td></tr>682</tbody></table></div>683684<h2 id="codes">Error codes</h2>685<p>The codes and HTTP statuses in this table are read from <code>x-error-codes</code> in the live <a href="/openapi.json">OpenAPI contract</a>.</p>686<div data-dyn="errors"></div>687688<h2 id="in-stream">Errors inside a stream</h2>689<p>When a streaming response fails after it started, the same envelope arrives as an SSE event, the HTTP status stays <code>200</code>, and the stream ends without <code>[DONE]</code>. See <a href="/docs/streaming#errors">Streaming</a>.</p>690691<h2 id="retries">Retrying</h2>692<ul>693 <li><strong>Do not retry <code>400</code>, <code>401</code>, <code>403</code>, <code>404</code>, <code>413</code>.</strong> The same request will fail the same way.</li>694 <li><strong>Retry <code>429</code> and <code>503</code> after the delay in <code>Retry-After</code>.</strong> The header is expressed in seconds. It is present on every <code>429</code> and on every <code>503</code> of type <code>overloaded_error</code>; if a <code>503</code> arrives without it (an <code>engine_error</code> while the engine connection is down), use backoff.</li>695 <li><strong>Do not loop on <code>insufficient_quota</code>.</strong> It is a <code>429</code>, but the wait can be hours. Surface it.</li>696 <li><strong>Retry <code>500</code>, <code>502</code>, <code>504</code> and network failures with exponential backoff and jitter</strong>: for example 1 s, 2 s, 4 s, 8 s, each multiplied by a random factor, capped at 30 s, and at most 4 or 5 attempts.</li>697 <li><strong><code>422 json_validation_failed</code></strong> may succeed on a new attempt because sampling varies, but the server already tried twice. Retry once at most.</li>698 <li><strong>Requests are not idempotent.</strong> A retry generates a new answer and consumes tokens again. A stream that failed midway cannot be continued.</li>699</ul>700<div data-sample="errors" data-label="Error handling with retries"></div>701702<h2 id="support">Reporting a problem</h2>703<p>Include the <code>request_id</code> (or the <code>x-request-id</code> header), the time, and the error <code>code</code>. The server keeps request metadata under that id, never the prompt or the answer.</p>`,704 },705706 // =====================================================================================707 {708 slug: "rate-limits", title: "Rate Limits", group: "Operations",709 lede: "Per-key limits protect each application; a bounded priority queue protects the machine.",710 html: () => r`711<h2 id="per-key">Limits attached to a key</h2>712<p>Each API key has its own limits, set by the administrator. A limit set to zero is disabled.</p>713<div class="table-wrap"><table class="data">714<thead><tr><th>Limit</th><th>How it is measured</th><th>When exceeded</th></tr></thead>715<tbody>716<tr><td><strong>Requests per minute</strong></td><td>Sliding 60-second window of accepted requests.</td><td><code>429 rate_limit_exceeded</code></td></tr>717<tr><td><strong>Tokens per minute</strong></td><td>Sliding 60-second window of tokens actually consumed (prompt plus completion). A new request is admitted only if its estimated cost, prompt plus <code>max_tokens</code>, still fits.</td><td><code>429 tokens_rate_limit_exceeded</code></td></tr>718<tr><td><strong>Daily token quota</strong></td><td>Tokens consumed since 00:00 UTC.</td><td><code>429 insufficient_quota</code></td></tr>719<tr><td><strong>Monthly token quota</strong></td><td>Tokens consumed since the first day of the month, UTC.</td><td><code>429 insufficient_quota</code></td></tr>720<tr><td><strong>Concurrency</strong></td><td>Requests of this key in flight at the same time, queued or generating.</td><td><code>429 concurrency_limit_exceeded</code></td></tr>721</tbody></table></div>722<p>Because the token-per-minute check uses <code>max_tokens</code> as the estimate, a realistic <code>max_tokens</code> lets more of your requests through than a very large one.</p>723724<h2 id="headers">Rate-limit headers</h2>725<p>Successful responses report where the key stands. A header pair is present only when the key has the corresponding limit.</p>726${code("http", "Response headers", `HTTP/1.1 200 OK727x-request-id: bf93c8bb18a5f0ccd0d22fc4728x-ratelimit-limit-requests: 60729x-ratelimit-remaining-requests: 59730x-ratelimit-limit-tokens: 200000731x-ratelimit-remaining-tokens: 198500`)}732<p>The numbers above are an example; yours depend on your key. Rejected requests carry <code>Retry-After</code>, in whole seconds: the time until the oldest request or token entry leaves the window, until the daily quota resets, or one second for a concurrency rejection.</p>733734<h2 id="429-vs-503">429 or 503?</h2>735<div class="table-wrap"><table class="data">736<thead><tr><th></th><th>429 Too Many Requests</th><th>503 Service Unavailable</th></tr></thead>737<tbody>738<tr><td><strong>Whose limit</strong></td><td>Your key's.</td><td>The server's capacity.</td></tr>739<tr><td><strong>Error type</strong></td><td><code>rate_limit_error</code></td><td><code>overloaded_error</code></td></tr>740<tr><td><strong>Codes</strong></td><td><code>rate_limit_exceeded</code>, <code>tokens_rate_limit_exceeded</code>, <code>concurrency_limit_exceeded</code>, <code>insufficient_quota</code></td><td><code>engine_overloaded</code>, <code>queue_timeout</code>, <code>engine_unavailable</code>, <code>memory_pressure</code></td></tr>741<tr><td><strong>What to do</strong></td><td>Slow down. Wait for <code>Retry-After</code>, lower your request rate or concurrency, or ask for a higher limit.</td><td>Back off. Wait for <code>Retry-After</code> and retry; nothing is wrong with your request.</td></tr>742</tbody></table></div>743744<h2 id="queue">Queueing and backpressure</h2>745<p>The engine generates for a bounded number of requests at once. Requests beyond that wait in a queue, and the queue itself is bounded:</p>746<ul>747 <li>If the queue is full, the request is rejected immediately with <code>503 engine_overloaded</code>. The server prefers a fast, explicit refusal to an unbounded wait.</li>748 <li>If a request waits longer than the queue timeout, it ends with <code>503 queue_timeout</code>.</li>749 <li>Under critical memory pressure, new requests are refused with <code>503 memory_pressure</code>.</li>750 <li>For streaming requests, the HTTP response starts only when the request leaves the queue, so these errors always arrive as ordinary HTTP errors, never in-stream.</li>751</ul>752<p>Server limits of this deployment, read from <code>x-limits</code> in the live <a href="/openapi.json">OpenAPI contract</a>:</p>753<div data-dyn="limits"></div>754755<h2 id="priority">Priority classes</h2>756<p>Each key belongs to one of four priority classes. When requests are waiting, the scheduler serves higher classes first:</p>757<div class="table-wrap"><table class="data">758<thead><tr><th>Class</th><th>Rank</th><th>Intended use</th></tr></thead>759<tbody>760<tr><td class="param-name">admin</td><td class="num">1 (highest)</td><td>Operator traffic, including the Playground when used with an administrator session.</td></tr>761<tr><td class="param-name">internal</td><td class="num">2</td><td>First-party services.</td></tr>762<tr><td class="param-name">client</td><td class="num">3</td><td>Interactive applications. The usual class for a key.</td></tr>763<tr><td class="param-name">background</td><td class="num">4</td><td>Batch jobs and anything that can wait.</td></tr>764</tbody></table></div>765<p><strong>Anti-starvation aging.</strong> Priority is not absolute. Each class starts with a fixed head start over the class below it, and a waiting request gains ground at a constant rate for as long as it waits. A background request that has waited long enough overtakes a newly arrived client request, so no class can be starved indefinitely. Among equals, service is first come, first served.</p>766<p>Priority only orders the queue. It never interrupts a generation in progress, and it does not raise a key's own limits.</p>`,767 },768769 // =====================================================================================770 {771 slug: "performance", title: "Performance", group: "Operations",772 lede: "How the engine serves concurrent requests, and the measurements published for this deployment.",773 html: () => r`774<h2 id="how">What determines latency</h2>775<ul>776 <li><strong>Queue time.</strong> Zero when the engine has capacity; otherwise governed by your key's <a href="/docs/rate-limits#priority">priority class</a>.</li>777 <li><strong>Time to first token.</strong> Mostly the time to process the prompt. It grows with prompt length and shrinks when the prompt shares a prefix with a recent request.</li>778 <li><strong>Generation speed.</strong> Tokens per second for your request, which depends on how many other generations share the GPU at that moment.</li>779</ul>780<h3 id="batching">Continuous batching</h3>781<p>The MLX engine does not process requests one after another. Generations that overlap in time are batched together on the GPU, and a new request can join while others are in progress. Each individual stream gets somewhat slower as concurrency rises, while the total number of tokens produced per second across all streams goes up.</p>782<h3 id="prefix-cache">KV prefix cache</h3>783<p>When a prompt starts with the same tokens as an earlier one (the same system prompt, or the earlier turns of a conversation), the engine reuses the attention state it already computed for that prefix instead of recomputing it. The number of prompt tokens served this way is reported in every response as <code>usage.prompt_tokens_details.cached_tokens</code>.</p>784785<h2 id="benchmarks">Published benchmarks</h2>786<div data-dyn="benchmarks"></div>787788<h2 id="tips">Getting the most out of it</h2>789<ul>790 <li><strong>Keep prefixes stable.</strong> Put the long, unchanging part of the prompt (instructions, tool definitions, reference text) first, and the part that varies last. Append to a conversation instead of rewriting its beginning.</li>791 <li><strong>Stream</strong> for anything a person reads; perceived latency becomes time to first token.</li>792 <li><strong>Set <code>max_tokens</code></strong> to what you need. It bounds latency and it is the estimate used by the tokens-per-minute check.</li>793 <li><strong>Measure with your own prompts.</strong> Request <code>include_usage</code> and time the first content delta and the final chunk, as the <a href="/playground">Playground</a> does.</li>794 <li><strong>Cancel what you no longer need.</strong> Closing the connection stops the generation and frees capacity.</li>795</ul>`,796 },797798 // =====================================================================================799 {800 slug: "changelog", title: "Changelog", group: "Resources", path: "/changelog",801 lede: "Changes to the API and to this portal, most recent first.",802 html: () => r`<div data-dyn="changelog"></div>`,803 },804];805806export const LINKS = [807 { title: "Status", href: "/status", group: "Resources", slug: "status" },808 { title: "Playground", href: "/playground", group: "Resources", slug: "playground" },809];810