SPB Git

Repository Public

openai-compatible.ts 15.3 KB 209 lines typescript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : Note SPBoucher AI8 * File             : src/lib/ai/openai-compatible.ts9 * Module           : lib10 * Repository       : https://git.spboucher.ai/note-spboucher-ai11 * Project Website  : https://note.spboucher.ai12 * Created          : 2026-11-1913 * Last Modified    : 2026-11-1914 *15 * Author           : Simon-Pierre Boucher16 * Position         : Professor of Finance17 * Institution      : Université du Québec en Outaouais (UQO)18 * Organization     : SP Boucher19 * Website          : https://spboucher.ai20 * Git              : https://git.spboucher.ai21 *22 * Preferred Contact: contact@spboucher.ai23 * Development      : dev@spboucher.ai24 * Support          : support@spboucher.ai25 * Security/Admin   : admin@spboucher.ai26 *27 * Copyright (c) 202628 * Simon-Pierre Boucher29 * All rights reserved where applicable. This file is governed by the30 * applicable project license, copyright notices and relevant law.31 * Existing third-party rights and notices remain unaffected.32 *33 * License          : None specified (no license file in this repository)34 *35 * Do not embed credentials, private keys, access tokens,36 * certificates or confidential personal information.37 *38 * =============================================================================39 * END OF SP BOUCHER SOURCE HEADER40 * =============================================================================41 */4243/**44 * Adapter for OpenAI-compatible endpoints (SPEC §3.5): Chat Completions over HTTP with SSE streaming. Serves OpenRouter, the MacLustr45 * LLM API and any other row of `ai_providers` with kind `openai_compatible` (base URL + NAME of the key variable).46 *47 * It speaks plain `fetch` rather than an SDK so that the wire behaviour is explicit and testable against a local fake server:48 *   - request: POST {base_url}/chat/completions, `stream: true`, `stream_options.include_usage`, `max_tokens`;49 *   - tools → `tools: [{type:"function", function:{…}}]`; tool-call deltas are accumulated and emitted once complete;50 *   - JSON schema → `response_format: {type:"json_schema", …strict}` when the model declares structured outputs, otherwise the schema51 *     is stated in the system prompt (+ `response_format: {type:"json_object"}` when the model declares JSON mode);52 *   - errors: HTTP status and the provider's `error.message` are surfaced (never the key); an `{"error":…}` frame inside the stream53 *     ends the call as an error; 408/429/5xx and network failures are retried (2 retries, backoff) only while nothing was received;54 *   - timeouts: `timeoutMs` for the whole call and an idle timeout between two frames, both reported as `timeout`.55 * Model capabilities come from the catalogue row (`ai_models.capabilities`, filled by discovery) — nothing is assumed.56 */57import type { AICallOptions, AIEvent, AIUsage } from "./types";58import { providerDef, providerKey, remoteModelId } from "./registry";5960export interface CompatCapabilities { tools?: boolean | null; structured_output?: boolean | null; json_mode?: boolean | null; vision?: boolean | null; reasoning?: boolean | null }61export interface CompatExtra { capabilities?: CompatCapabilities | null; idleTimeoutMs?: number; retries?: number; fetchImpl?: typeof fetch }62const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);63const now = () => performance.now();64const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((res, rej) => { const t = setTimeout(res, ms); signal?.addEventListener("abort", () => { clearTimeout(t); rej(new Error("aborted")); }, { once: true }); });65class TimeoutError extends Error { constructor(what: string) { super(`timeout: ${what}`); } }6667/** Body of a Chat Completions request for these options (exported for tests: this is the exact wire contract). */68export function buildChatCompletionsBody(o: AICallOptions, remoteId: string, caps: CompatCapabilities = {}): Record<string, unknown> {69  const messages: unknown[] = [];70  let system = o.system ?? "";71  const body: Record<string, unknown> = { model: remoteId, stream: true, stream_options: { include_usage: true }, max_tokens: o.maxTokens ?? 8000 };72  if (o.jsonSchema) {73    if (caps.structured_output) body.response_format = { type: "json_schema", json_schema: { name: o.jsonSchema.name, schema: o.jsonSchema.schema, strict: true } };74    else {75      system = `${system}${system ? "\n\n" : ""}Return ONLY one JSON object, with no prose and no Markdown fence, that validates against this JSON Schema:\n${JSON.stringify(o.jsonSchema.schema)}`;76      if (caps.json_mode) body.response_format = { type: "json_object" };77    }78  }79  if (system) messages.push({ role: "system", content: system });80  for (const m of o.messages) {81    if (m.role === "user" && m.images?.length) messages.push({ role: "user", content: [...m.images.map((im) => ({ type: "image_url", image_url: { url: `data:${im.mime};base64,${im.dataBase64}` } })), { type: "text", text: m.content }] });82    else messages.push({ role: m.role, content: m.content });83  }84  body.messages = messages;85  if (o.tools?.length) body.tools = o.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.parameters } }));86  if (o.effort && caps.reasoning) body.reasoning_effort = o.effort;87  return body;88}8990async function errorMessage(res: Response): Promise<string> {91  const raw = (await res.text().catch(() => "")).slice(0, 2000);92  try { const j = JSON.parse(raw); const m = j?.error?.message ?? j?.error ?? j?.message ?? j?.detail; if (typeof m === "string" && m) return m.slice(0, 400); if (m) return JSON.stringify(m).slice(0, 400); } catch { /* not JSON */ }93  return raw.replace(/\s+/g, " ").slice(0, 300) || res.statusText || `HTTP ${res.status}`;94}9596/** Split a byte stream into SSE `data:` payloads (comments and other fields are ignored), enforcing the idle timeout. */97async function* sseData(body: ReadableStream<Uint8Array>, idleMs: number): AsyncGenerator<string> {98  const reader = body.getReader(); const dec = new TextDecoder(); let buf = "";99  try {100    for (;;) {101      let timer: ReturnType<typeof setTimeout> | undefined;102      const idle = new Promise<never>((_, rej) => { timer = setTimeout(() => rej(new TimeoutError(`no data for ${Math.round(idleMs / 1000)} s`)), idleMs); });103      let chunk: ReadableStreamReadResult<Uint8Array>;104      try { chunk = await Promise.race([reader.read(), idle]); } finally { clearTimeout(timer); }105      if (chunk.done) break;106      buf += dec.decode(chunk.value, { stream: true });107      let i: number;108      while ((i = buf.search(/\r?\n/)) >= 0) {109        const line = buf.slice(0, i); buf = buf.slice(i + (buf[i] === "\r" ? 2 : 1));110        if (line.startsWith("data:")) yield line.slice(5).trimStart();111      }112    }113    if (buf.startsWith("data:")) yield buf.slice(5).trimStart();114  } finally { reader.cancel().catch(() => {}); }115}116117export async function* streamOpenAICompatible(o: AICallOptions, extra: CompatExtra = {}): AsyncGenerator<AIEvent> {118  const t0 = now(); let ttft: number | null = null;119  const def = providerDef(o.provider);120  if (!def || def.kind !== "openai_compatible" || !def.baseUrl) { yield { type: "error", message: `Provider "${o.provider}" is not an OpenAI-compatible endpoint with a base URL.` }; return; }121  const key = providerKey(o.provider);122  if (!key) { yield { type: "error", message: `Provider "${def.name}" is not configured: the environment variable ${def.keyEnv ?? "(none)"} is not set.`, status: 401 }; return; }123  const totalMs = o.timeoutMs ?? 10 * 60_000; const idleMs = extra.idleTimeoutMs ?? 120_000; const maxRetries = extra.retries ?? 2;124  const doFetch = extra.fetchImpl ?? fetch;125  const timeout = new AbortController(); const timer = setTimeout(() => timeout.abort(), totalMs);126  const signal = o.signal ? AbortSignal.any([o.signal, timeout.signal]) : timeout.signal;127  const body = JSON.stringify(buildChatCompletionsBody(o, remoteModelId(o.provider, o.model), extra.capabilities ?? {}));128  try {129    let res: Response | null = null; let lastErr: { message: string; status?: number } | null = null;130    for (let attempt = 0; attempt <= maxRetries; attempt++) {131      if (attempt) await sleep(Math.min(4000, 300 * 2 ** (attempt - 1)), signal);132      try {133        const r = await doFetch(`${def.baseUrl}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", accept: "text/event-stream", authorization: `Bearer ${key}` }, body, signal });134        if (r.ok && r.body) { res = r; break; }135        lastErr = { message: await errorMessage(r), status: r.status };136        if (!RETRYABLE.has(r.status)) break;137      } catch (e) {138        if (signal.aborted) throw e;139        lastErr = { message: `network error: ${String((e as Error)?.message ?? e).slice(0, 200)}` };140      }141    }142    if (!res) { yield { type: "error", message: lastErr?.message ?? "no response", status: lastErr?.status }; return; }143144    let text = ""; let stop: string | null = null; let refusal = ""; let u: any = null;145    const calls = new Map<number, { id: string; name: string; args: string }>();146    for await (const data of sseData(res.body!, idleMs)) {147      if (data === "[DONE]") break;148      let j: any; try { j = JSON.parse(data); } catch { continue; }149      if (j?.error) { const m = typeof j.error === "string" ? j.error : j.error.message ?? JSON.stringify(j.error); yield { type: "error", message: String(m).slice(0, 400), status: typeof j.error?.code === "number" ? j.error.code : undefined }; return; }150      if (j?.usage) u = j.usage;151      const ch = j?.choices?.[0]; if (!ch) continue;152      const d = ch.delta ?? {};153      if (typeof d.content === "string" && d.content) { if (ttft === null) ttft = now() - t0; text += d.content; yield { type: "text", delta: d.content }; }154      else if ((d.reasoning || d.reasoning_content || d.tool_calls) && ttft === null) ttft = now() - t0;155      if (typeof d.refusal === "string") refusal += d.refusal;156      for (const tc of d.tool_calls ?? []) {157        const idx = typeof tc.index === "number" ? tc.index : calls.size;158        const cur = calls.get(idx) ?? { id: "", name: "", args: "" };159        if (tc.id) cur.id = tc.id; if (tc.function?.name) cur.name = tc.function.name; if (typeof tc.function?.arguments === "string") cur.args += tc.function.arguments;160        calls.set(idx, cur);161      }162      if (ch.finish_reason) stop = ch.finish_reason;163    }164    for (const [idx, c] of [...calls.entries()].sort((a, b) => a[0] - b[0])) { let args: unknown = c.args; try { args = JSON.parse(c.args || "{}"); } catch { /* keep the raw string */ } yield { type: "tool_call", id: c.id || `call_${idx}`, name: c.name, args }; }165    if (stop === null && !text && !calls.size) { yield { type: "error", message: "The provider closed the stream without an answer." }; return; }166    const usage: AIUsage = { inputTokens: u?.prompt_tokens ?? 0, outputTokens: u?.completion_tokens ?? 0, cachedTokens: u?.prompt_tokens_details?.cached_tokens ?? 0, reasoningTokens: u?.completion_tokens_details?.reasoning_tokens ?? 0 };167    yield { type: "done", usage, stop, ttftMs: ttft, totalMs: now() - t0, refusal: stop === "content_filter" ? "content filter" : refusal ? "refusal" : undefined };168  } catch (e) {169    if (o.signal?.aborted) { yield { type: "error", message: "aborted" }; return; }170    if (timeout.signal.aborted) { yield { type: "error", message: `timeout: no complete answer within ${Math.round(totalMs / 1000)} s`, status: 408 }; return; }171    if (e instanceof TimeoutError) { yield { type: "error", message: e.message, status: 408 }; return; }172    yield { type: "error", message: String((e as Error)?.message ?? e).slice(0, 500) };173  } finally { clearTimeout(timer); }174}175176// ---------- catalogue discovery through GET {base_url}/models177export interface DiscoveredModel { remoteId: string; name: string; contextWindow: number | null; maxOutput: number | null; inputPrice: number | null; cachedPrice: number | null; outputPrice: number | null; capabilities: CompatCapabilities & { modalities?: string[]; source: string }; textOutput: boolean; created: Date | null }178const perMillion = (v: unknown): number | null => { if (v == null || v === "") return null; const n = Number(v); return Number.isFinite(n) && n >= 0 ? Math.round(n * 1e6 * 1e6) / 1e6 : null; };179/**180 * Normalise one entry of a `/models` list. OpenRouter documents `context_length`, `pricing.{prompt,completion,input_cache_read}` (USD per181 * token, as strings), `architecture.{input_modalities,output_modalities}`, `supported_parameters`, `top_provider.max_completion_tokens`.182 * A bare OpenAI-style entry (`{id, object:"model"}`, e.g. mlx_lm.server) yields nulls: unknown stays unknown.183 */184export function normaliseModelEntry(m: any): DiscoveredModel | null {185  if (!m || typeof m.id !== "string" || !m.id.trim()) return null;186  const sp: string[] | null = Array.isArray(m.supported_parameters) ? m.supported_parameters : null;187  const inMod: string[] | null = Array.isArray(m.architecture?.input_modalities) ? m.architecture.input_modalities : null;188  const outMod: string[] | null = Array.isArray(m.architecture?.output_modalities) ? m.architecture.output_modalities : null;189  const has = (p: string) => (sp ? sp.includes(p) : null);190  const ctx = Number(m.context_length ?? m.context_window ?? m.max_context_length); const maxOut = Number(m.top_provider?.max_completion_tokens ?? m.max_output_tokens ?? m.max_completion_tokens);191  return {192    remoteId: m.id.trim(), name: typeof m.name === "string" && m.name.trim() ? m.name.trim() : m.id.trim(),193    contextWindow: Number.isFinite(ctx) && ctx > 0 ? Math.round(ctx) : null, maxOutput: Number.isFinite(maxOut) && maxOut > 0 ? Math.round(maxOut) : null,194    inputPrice: perMillion(m.pricing?.prompt), cachedPrice: perMillion(m.pricing?.input_cache_read), outputPrice: perMillion(m.pricing?.completion),195    capabilities: { tools: has("tools"), structured_output: has("structured_outputs"), json_mode: has("response_format"), vision: inMod ? inMod.includes("image") : null, reasoning: sp ? sp.includes("reasoning") || sp.includes("include_reasoning") : null, ...(inMod ? { modalities: inMod } : {}), source: "provider /models" },196    textOutput: outMod ? outMod.includes("text") : true,197    created: typeof m.created === "number" && m.created > 0 ? new Date(m.created * 1000) : null,198  };199}200/** List the models of an OpenAI-compatible endpoint. `key` may be null for a public catalogue (OpenRouter's `/models` is keyless). */201export async function fetchModelList(baseUrl: string, key: string | null, opts: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}): Promise<DiscoveredModel[]> {202  const res = await (opts.fetchImpl ?? fetch)(`${baseUrl.replace(/\/+$/, "")}/models`, { headers: { accept: "application/json", ...(key ? { authorization: `Bearer ${key}` } : {}) }, signal: AbortSignal.timeout(opts.timeoutMs ?? 30_000) });203  if (!res.ok) throw Object.assign(new Error(`GET /models → HTTP ${res.status}: ${await errorMessage(res)}`), { status: res.status });204  const j: any = await res.json(); const list: unknown[] = Array.isArray(j) ? j : Array.isArray(j?.data) ? j.data : Array.isArray(j?.models) ? j.models : [];205  const out: DiscoveredModel[] = []; const seen = new Set<string>();206  for (const m of list) { const n = normaliseModelEntry(m); if (n && !seen.has(n.remoteId)) { seen.add(n.remoteId); out.push(n); } }207  return out;208}209
Files main
Menu

Theme