Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : Note SPBoucher AI8 * File : src/lib/ai/openai-compatible.ts9 * Module : lib10 * Repository : https://git.spboucher.ai/note-spboucher-ai11 * Project Website : https://note.spboucher.ai12 * Created : 2026-11-1913 * Last Modified : 2026-11-1914 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: contact@spboucher.ai23 * Development : dev@spboucher.ai24 * Support : support@spboucher.ai25 * Security/Admin : admin@spboucher.ai26 *27 * Copyright (c) 202628 * Simon-Pierre Boucher29 * All rights reserved where applicable. This file is governed by the30 * applicable project license, copyright notices and relevant law.31 * Existing third-party rights and notices remain unaffected.32 *33 * License : None specified (no license file in this repository)34 *35 * Do not embed credentials, private keys, access tokens,36 * certificates or confidential personal information.37 *38 * =============================================================================39 * END OF SP BOUCHER SOURCE HEADER40 * =============================================================================41 */4243/**44 * Adapter for OpenAI-compatible endpoints (SPEC §3.5): Chat Completions over HTTP with SSE streaming. Serves OpenRouter, the MacLustr45 * LLM API and any other row of `ai_providers` with kind `openai_compatible` (base URL + NAME of the key variable).46 *47 * It speaks plain `fetch` rather than an SDK so that the wire behaviour is explicit and testable against a local fake server:48 * - request: POST {base_url}/chat/completions, `stream: true`, `stream_options.include_usage`, `max_tokens`;49 * - tools → `tools: [{type:"function", function:{…}}]`; tool-call deltas are accumulated and emitted once complete;50 * - JSON schema → `response_format: {type:"json_schema", …strict}` when the model declares structured outputs, otherwise the schema51 * is stated in the system prompt (+ `response_format: {type:"json_object"}` when the model declares JSON mode);52 * - errors: HTTP status and the provider's `error.message` are surfaced (never the key); an `{"error":…}` frame inside the stream53 * ends the call as an error; 408/429/5xx and network failures are retried (2 retries, backoff) only while nothing was received;54 * - timeouts: `timeoutMs` for the whole call and an idle timeout between two frames, both reported as `timeout`.55 * Model capabilities come from the catalogue row (`ai_models.capabilities`, filled by discovery) — nothing is assumed.56 */57import type { AICallOptions, AIEvent, AIUsage } from "./types";58import { providerDef, providerKey, remoteModelId } from "./registry";5960export interface CompatCapabilities { tools?: boolean | null; structured_output?: boolean | null; json_mode?: boolean | null; vision?: boolean | null; reasoning?: boolean | null }61export interface CompatExtra { capabilities?: CompatCapabilities | null; idleTimeoutMs?: number; retries?: number; fetchImpl?: typeof fetch }62const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);63const now = () => performance.now();64const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((res, rej) => { const t = setTimeout(res, ms); signal?.addEventListener("abort", () => { clearTimeout(t); rej(new Error("aborted")); }, { once: true }); });65class TimeoutError extends Error { constructor(what: string) { super(`timeout: ${what}`); } }6667/** Body of a Chat Completions request for these options (exported for tests: this is the exact wire contract). */68export function buildChatCompletionsBody(o: AICallOptions, remoteId: string, caps: CompatCapabilities = {}): Record<string, unknown> {69 const messages: unknown[] = [];70 let system = o.system ?? "";71 const body: Record<string, unknown> = { model: remoteId, stream: true, stream_options: { include_usage: true }, max_tokens: o.maxTokens ?? 8000 };72 if (o.jsonSchema) {73 if (caps.structured_output) body.response_format = { type: "json_schema", json_schema: { name: o.jsonSchema.name, schema: o.jsonSchema.schema, strict: true } };74 else {75 system = `${system}${system ? "\n\n" : ""}Return ONLY one JSON object, with no prose and no Markdown fence, that validates against this JSON Schema:\n${JSON.stringify(o.jsonSchema.schema)}`;76 if (caps.json_mode) body.response_format = { type: "json_object" };77 }78 }79 if (system) messages.push({ role: "system", content: system });80 for (const m of o.messages) {81 if (m.role === "user" && m.images?.length) messages.push({ role: "user", content: [...m.images.map((im) => ({ type: "image_url", image_url: { url: `data:${im.mime};base64,${im.dataBase64}` } })), { type: "text", text: m.content }] });82 else messages.push({ role: m.role, content: m.content });83 }84 body.messages = messages;85 if (o.tools?.length) body.tools = o.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }));86 if (o.effort && caps.reasoning) body.reasoning_effort = o.effort;87 return body;88}8990async function errorMessage(res: Response): Promise<string> {91 const raw = (await res.text().catch(() => "")).slice(0, 2000);92 try { const j = JSON.parse(raw); const m = j?.error?.message ?? j?.error ?? j?.message ?? j?.detail; if (typeof m === "string" && m) return m.slice(0, 400); if (m) return JSON.stringify(m).slice(0, 400); } catch { /* not JSON */ }93 return raw.replace(/\s+/g, " ").slice(0, 300) || res.statusText || `HTTP ${res.status}`;94}9596/** Split a byte stream into SSE `data:` payloads (comments and other fields are ignored), enforcing the idle timeout. */97async function* sseData(body: ReadableStream<Uint8Array>, idleMs: number): AsyncGenerator<string> {98 const reader = body.getReader(); const dec = new TextDecoder(); let buf = "";99 try {100 for (;;) {101 let timer: ReturnType<typeof setTimeout> | undefined;102 const idle = new Promise<never>((_, rej) => { timer = setTimeout(() => rej(new TimeoutError(`no data for ${Math.round(idleMs / 1000)} s`)), idleMs); });103 let chunk: ReadableStreamReadResult<Uint8Array>;104 try { chunk = await Promise.race([reader.read(), idle]); } finally { clearTimeout(timer); }105 if (chunk.done) break;106 buf += dec.decode(chunk.value, { stream: true });107 let i: number;108 while ((i = buf.search(/\r?\n/)) >= 0) {109 const line = buf.slice(0, i); buf = buf.slice(i + (buf[i] === "\r" ? 2 : 1));110 if (line.startsWith("data:")) yield line.slice(5).trimStart();111 }112 }113 if (buf.startsWith("data:")) yield buf.slice(5).trimStart();114 } finally { reader.cancel().catch(() => {}); }115}116117export async function* streamOpenAICompatible(o: AICallOptions, extra: CompatExtra = {}): AsyncGenerator<AIEvent> {118 const t0 = now(); let ttft: number | null = null;119 const def = providerDef(o.provider);120 if (!def || def.kind !== "openai_compatible" || !def.baseUrl) { yield { type: "error", message: `Provider "${o.provider}" is not an OpenAI-compatible endpoint with a base URL.` }; return; }121 const key = providerKey(o.provider);122 if (!key) { yield { type: "error", message: `Provider "${def.name}" is not configured: the environment variable ${def.keyEnv ?? "(none)"} is not set.`, status: 401 }; return; }123 const totalMs = o.timeoutMs ?? 10 * 60_000; const idleMs = extra.idleTimeoutMs ?? 120_000; const maxRetries = extra.retries ?? 2;124 const doFetch = extra.fetchImpl ?? fetch;125 const timeout = new AbortController(); const timer = setTimeout(() => timeout.abort(), totalMs);126 const signal = o.signal ? AbortSignal.any([o.signal, timeout.signal]) : timeout.signal;127 const body = JSON.stringify(buildChatCompletionsBody(o, remoteModelId(o.provider, o.model), extra.capabilities ?? {}));128 try {129 let res: Response | null = null; let lastErr: { message: string; status?: number } | null = null;130 for (let attempt = 0; attempt <= maxRetries; attempt++) {131 if (attempt) await sleep(Math.min(4000, 300 * 2 ** (attempt - 1)), signal);132 try {133 const r = await doFetch(`${def.baseUrl}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", accept: "text/event-stream", authorization: `Bearer ${key}` }, body, signal });134 if (r.ok && r.body) { res = r; break; }135 lastErr = { message: await errorMessage(r), status: r.status };136 if (!RETRYABLE.has(r.status)) break;137 } catch (e) {138 if (signal.aborted) throw e;139 lastErr = { message: `network error: ${String((e as Error)?.message ?? e).slice(0, 200)}` };140 }141 }142 if (!res) { yield { type: "error", message: lastErr?.message ?? "no response", status: lastErr?.status }; return; }143144 let text = ""; let stop: string | null = null; let refusal = ""; let u: any = null;145 const calls = new Map<number, { id: string; name: string; args: string }>();146 for await (const data of sseData(res.body!, idleMs)) {147 if (data === "[DONE]") break;148 let j: any; try { j = JSON.parse(data); } catch { continue; }149 if (j?.error) { const m = typeof j.error === "string" ? j.error : j.error.message ?? JSON.stringify(j.error); yield { type: "error", message: String(m).slice(0, 400), status: typeof j.error?.code === "number" ? j.error.code : undefined }; return; }150 if (j?.usage) u = j.usage;151 const ch = j?.choices?.[0]; if (!ch) continue;152 const d = ch.delta ?? {};153 if (typeof d.content === "string" && d.content) { if (ttft === null) ttft = now() - t0; text += d.content; yield { type: "text", delta: d.content }; }154 else if ((d.reasoning || d.reasoning_content || d.tool_calls) && ttft === null) ttft = now() - t0;155 if (typeof d.refusal === "string") refusal += d.refusal;156 for (const tc of d.tool_calls ?? []) {157 const idx = typeof tc.index === "number" ? tc.index : calls.size;158 const cur = calls.get(idx) ?? { id: "", name: "", args: "" };159 if (tc.id) cur.id = tc.id; if (tc.function?.name) cur.name = tc.function.name; if (typeof tc.function?.arguments === "string") cur.args += tc.function.arguments;160 calls.set(idx, cur);161 }162 if (ch.finish_reason) stop = ch.finish_reason;163 }164 for (const [idx, c] of [...calls.entries()].sort((a, b) => a[0] - b[0])) { let args: unknown = c.args; try { args = JSON.parse(c.args || "{}"); } catch { /* keep the raw string */ } yield { type: "tool_call", id: c.id || `call_${idx}`, name: c.name, args }; }165 if (stop === null && !text && !calls.size) { yield { type: "error", message: "The provider closed the stream without an answer." }; return; }166 const usage: AIUsage = { inputTokens: u?.prompt_tokens ?? 0, outputTokens: u?.completion_tokens ?? 0, cachedTokens: u?.prompt_tokens_details?.cached_tokens ?? 0, reasoningTokens: u?.completion_tokens_details?.reasoning_tokens ?? 0 };167 yield { type: "done", usage, stop, ttftMs: ttft, totalMs: now() - t0, refusal: stop === "content_filter" ? "content filter" : refusal ? "refusal" : undefined };168 } catch (e) {169 if (o.signal?.aborted) { yield { type: "error", message: "aborted" }; return; }170 if (timeout.signal.aborted) { yield { type: "error", message: `timeout: no complete answer within ${Math.round(totalMs / 1000)} s`, status: 408 }; return; }171 if (e instanceof TimeoutError) { yield { type: "error", message: e.message, status: 408 }; return; }172 yield { type: "error", message: String((e as Error)?.message ?? e).slice(0, 500) };173 } finally { clearTimeout(timer); }174}175176// ---------- catalogue discovery through GET {base_url}/models177export interface DiscoveredModel { remoteId: string; name: string; contextWindow: number | null; maxOutput: number | null; inputPrice: number | null; cachedPrice: number | null; outputPrice: number | null; capabilities: CompatCapabilities & { modalities?: string[]; source: string }; textOutput: boolean; created: Date | null }178const perMillion = (v: unknown): number | null => { if (v == null || v === "") return null; const n = Number(v); return Number.isFinite(n) && n >= 0 ? Math.round(n * 1e6 * 1e6) / 1e6 : null; };179/**180 * Normalise one entry of a `/models` list. OpenRouter documents `context_length`, `pricing.{prompt,completion,input_cache_read}` (USD per181 * token, as strings), `architecture.{input_modalities,output_modalities}`, `supported_parameters`, `top_provider.max_completion_tokens`.182 * A bare OpenAI-style entry (`{id, object:"model"}`, e.g. mlx_lm.server) yields nulls: unknown stays unknown.183 */184export function normaliseModelEntry(m: any): DiscoveredModel | null {185 if (!m || typeof m.id !== "string" || !m.id.trim()) return null;186 const sp: string[] | null = Array.isArray(m.supported_parameters) ? m.supported_parameters : null;187 const inMod: string[] | null = Array.isArray(m.architecture?.input_modalities) ? m.architecture.input_modalities : null;188 const outMod: string[] | null = Array.isArray(m.architecture?.output_modalities) ? m.architecture.output_modalities : null;189 const has = (p: string) => (sp ? sp.includes(p) : null);190 const ctx = Number(m.context_length ?? m.context_window ?? m.max_context_length); const maxOut = Number(m.top_provider?.max_completion_tokens ?? m.max_output_tokens ?? m.max_completion_tokens);191 return {192 remoteId: m.id.trim(), name: typeof m.name === "string" && m.name.trim() ? m.name.trim() : m.id.trim(),193 contextWindow: Number.isFinite(ctx) && ctx > 0 ? Math.round(ctx) : null, maxOutput: Number.isFinite(maxOut) && maxOut > 0 ? Math.round(maxOut) : null,194 inputPrice: perMillion(m.pricing?.prompt), cachedPrice: perMillion(m.pricing?.input_cache_read), outputPrice: perMillion(m.pricing?.completion),195 capabilities: { tools: has("tools"), structured_output: has("structured_outputs"), json_mode: has("response_format"), vision: inMod ? inMod.includes("image") : null, reasoning: sp ? sp.includes("reasoning") || sp.includes("include_reasoning") : null, ...(inMod ? { modalities: inMod } : {}), source: "provider /models" },196 textOutput: outMod ? outMod.includes("text") : true,197 created: typeof m.created === "number" && m.created > 0 ? new Date(m.created * 1000) : null,198 };199}200/** List the models of an OpenAI-compatible endpoint. `key` may be null for a public catalogue (OpenRouter's `/models` is keyless). */201export async function fetchModelList(baseUrl: string, key: string | null, opts: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}): Promise<DiscoveredModel[]> {202 const res = await (opts.fetchImpl ?? fetch)(`${baseUrl.replace(/\/+$/, "")}/models`, { headers: { accept: "application/json", ...(key ? { authorization: `Bearer ${key}` } : {}) }, signal: AbortSignal.timeout(opts.timeoutMs ?? 30_000) });203 if (!res.ok) throw Object.assign(new Error(`GET /models → HTTP ${res.status}: ${await errorMessage(res)}`), { status: res.status });204 const j: any = await res.json(); const list: unknown[] = Array.isArray(j) ? j : Array.isArray(j?.data) ? j.data : Array.isArray(j?.models) ? j.models : [];205 const out: DiscoveredModel[] = []; const seen = new Set<string>();206 for (const m of list) { const n = normaliseModelEntry(m); if (n && !seen.has(n.remoteId)) { seen.add(n.remoteId); out.push(n); } }207 return out;208}209