import type OpenAI from "openai"; import { type AIProviderAdapter, type PolyModel, PolyProviderError } from "@/lib/ai/core/types"; import { createOpenAICompatAdapter } from "../shared/openai-compat/factory"; /** * Custom OpenAI-compatible endpoints (Ollama, LM Studio, vLLM, llama.cpp server, MLX-LM, TGI, * LocalAI, any `/v1/chat/completions` server). One adapter instance per endpoint, built at request * time from the user's stored configuration (`@/lib/endpoints/service`). * * Model keys are `custom/:` — the provider prefix stays `custom` so every * `parseModelKey()` call site keeps working; the endpoint id and the (possibly `:`-containing) * model id are split by `parseCustomModelKey()` on the first `:` after the prefix. */ export const CUSTOM_KEY_PREFIX = "custom/"; /** Sent as `Authorization: Bearer …` when the endpoint has no key (Ollama & co. ignore it). */ export const NO_KEY_SENTINEL = "polyllm-no-key"; export interface CustomEndpointConfig { id: string; name: string; /** Up to and including `/v1` (e.g. http://localhost:11434/v1). No trailing slash. */ baseUrl: string; apiKey?: string | null; headers?: Record | null; /** Relative path appended to `baseUrl` for discovery (default `/models`). Empty = discovery disabled. */ modelsPath?: string | null; /** Manual declarations (also act as capability overrides for discovered ids). */ manualModels?: ManualModel[] | null; /** Discovery request timeout (ms). */ timeoutMs?: number; } export interface ManualModel { id: string; displayName?: string; contextTokens?: number; vision?: boolean; tools?: boolean; reasoning?: boolean; } export function customModelKey(endpointId: string, modelId: string): string { return `${CUSTOM_KEY_PREFIX}${endpointId}:${modelId}`; } /** `custom/cep_abc:llama3.1:8b` → `{ endpointId: "cep_abc", modelId: "llama3.1:8b" }`. */ export function parseCustomModelKey(key: string): { endpointId: string; modelId: string } | null { if (!key.startsWith(CUSTOM_KEY_PREFIX)) return null; const rest = key.slice(CUSTOM_KEY_PREFIX.length); const idx = rest.indexOf(":"); if (idx <= 0 || idx === rest.length - 1) return null; return { endpointId: rest.slice(0, idx), modelId: rest.slice(idx + 1) }; } export function isCustomModelKey(key: string): boolean { return parseCustomModelKey(key) !== null; } export function normalizeBaseUrl(url: string): string { return url.trim().replace(/\/+$/, ""); } export function normalizeModelsPath(p: string | null | undefined): string { const t = (p ?? "").trim(); if (!t) return ""; return t.startsWith("/") ? t : `/${t}`; } export interface DiscoveredModel { id: string; owned_by?: string; created?: number; /** Some servers (LM Studio, llama.cpp) add metadata; kept opaque. */ [k: string]: unknown; } /** Build a PolyModel for one model id on an endpoint; `manual` supplies capability overrides. */ export function toCustomPolyModel(cfg: Pick, modelId: string, manual?: ManualModel, extra?: { ownedBy?: string; created?: number; source: "discovery" | "manual" }): PolyModel { return { key: customModelKey(cfg.id, modelId), id: modelId, provider: "custom", displayName: manual?.displayName?.trim() || modelId, family: cfg.name, capabilities: { text: true, vision: manual?.vision ?? false, audioInput: false, audioOutput: false, imageGeneration: false, video: false, reasoning: manual?.reasoning ?? false, tools: manual?.tools ?? false, structuredOutput: false, streaming: true, files: false, webSearch: false, }, limits: manual?.contextTokens ? { contextTokens: manual.contextTokens } : {}, parameters: { temperature: true, topP: true, maxTokens: true, stop: true, seed: true, frequencyPenalty: true, presencePenalty: true, temperatureRange: { min: 0, max: 2 }, }, status: "active", // Cost is unknown for arbitrary endpoints (local = free, hosted = billed elsewhere): never invent a price. pricing: null, metadata: { endpointId: cfg.id, endpointName: cfg.name, ownedBy: extra?.ownedBy, createdAt: extra?.created ? new Date(extra.created * 1000).toISOString() : undefined, source: extra?.source ?? "manual", sortWeight: -40, }, }; } /** Merge discovered ids with manual declarations (manual entries win on capabilities; unknown ids are appended). */ export function mergeCustomModels(cfg: Pick, discovered: DiscoveredModel[], manual: ManualModel[] = []): PolyModel[] { const manualById = new Map(manual.filter((m) => m.id?.trim()).map((m) => [m.id.trim(), m])); const seen = new Set(); const out: PolyModel[] = []; for (const d of discovered) { if (!d?.id || typeof d.id !== "string" || seen.has(d.id)) continue; seen.add(d.id); out.push(toCustomPolyModel(cfg, d.id, manualById.get(d.id), { ownedBy: typeof d.owned_by === "string" ? d.owned_by : undefined, created: typeof d.created === "number" ? d.created : undefined, source: "discovery" })); } for (const m of manual) { const id = m.id?.trim(); if (!id || seen.has(id)) continue; seen.add(id); out.push(toCustomPolyModel(cfg, id, m, { source: "manual" })); } return out; } function authHeaders(cfg: CustomEndpointConfig): Record { const h: Record = { Accept: "application/json", ...(cfg.headers ?? {}) }; if (cfg.apiKey && cfg.apiKey !== NO_KEY_SENTINEL) h.Authorization = `Bearer ${cfg.apiKey}`; return h; } /** `GET {baseUrl}{modelsPath}` — accepts OpenAI `{data:[{id}]}`, bare arrays and Ollama-style `{models:[{name}]}` bodies. */ export async function discoverCustomModels(cfg: CustomEndpointConfig, signal?: AbortSignal): Promise { const path = normalizeModelsPath(cfg.modelsPath ?? "/models"); if (!path) return []; const url = `${normalizeBaseUrl(cfg.baseUrl)}${path}`; const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), cfg.timeoutMs ?? 8_000); signal?.addEventListener("abort", () => ctrl.abort(), { once: true }); try { const res = await fetch(url, { headers: authHeaders(cfg), signal: ctrl.signal, redirect: "manual" }); if (!res.ok) { const text = await res.text().catch(() => ""); let message = `HTTP ${res.status}`; try { const body = JSON.parse(text) as { error?: { message?: string } | string; message?: string }; message = (typeof body.error === "object" ? body.error?.message : typeof body.error === "string" ? body.error : undefined) ?? body.message ?? message; } catch { if (text) message = `${message}: ${text.slice(0, 200)}`; } throw Object.assign(new Error(message), { status: res.status, headers: res.headers }); } const body = (await res.json()) as unknown; return parseModelListing(body); } finally { clearTimeout(timer); } } export function parseModelListing(body: unknown): DiscoveredModel[] { const pick = (arr: unknown[]): DiscoveredModel[] => arr .map((m) => { if (typeof m === "string") return { id: m }; if (m && typeof m === "object") { const o = m as Record; const id = typeof o.id === "string" ? o.id : typeof o.name === "string" ? o.name : typeof o.model === "string" ? o.model : null; return id ? { ...o, id } : null; } return null; }) .filter((m): m is DiscoveredModel => Boolean(m)); if (Array.isArray(body)) return pick(body); if (body && typeof body === "object") { const o = body as Record; if (Array.isArray(o.data)) return pick(o.data); if (Array.isArray(o.models)) return pick(o.models); } return []; } /** * Adapter for one endpoint. Chat goes through the shared Chat Completions implementation with the * endpoint's base URL and headers; discovery uses `discoverCustomModels`. */ export function createCustomEndpointAdapter(cfg: CustomEndpointConfig): AIProviderAdapter { const listModels = async (_client: OpenAI, _apiKey: string, signal?: AbortSignal): Promise => { const discovered = await discoverCustomModels(cfg, signal); return mergeCustomModels(cfg, discovered, cfg.manualModels ?? []); }; return createOpenAICompatAdapter({ id: "custom", name: cfg.name, baseURL: normalizeBaseUrl(cfg.baseUrl), keyDocsUrl: "", defaultHeaders: cfg.headers ?? undefined, listModels, // Most local servers only understand `max_tokens`; the newer `max_completion_tokens` 400s on several of them. useMaxTokens: true, messageOptions: { inlineFiles: true }, tweakParams: (params, settings, req) => { const p = params as unknown as Record; // Endpoints marked "reasoning" by the user follow the de-facto `reasoning_effort` field (vLLM, llama.cpp, LM Studio). if (req.modelInfo?.capabilities.reasoning && settings.reasoningEffort && settings.reasoningEffort !== "none") p.reasoning_effort = settings.reasoningEffort === "minimal" ? "low" : settings.reasoningEffort; }, refineError: (status, _code, message) => { if (status === 404 && /model/i.test(message)) return "MODEL_NOT_FOUND"; if (status === 401 || status === 403) return "INVALID_API_KEY"; if (status === 503 || status === 502) return "PROVIDER_UNAVAILABLE"; return undefined; }, timeoutMs: 10 * 60_000, }); } /** * Registered under `ADAPTERS.custom` so `getAdapter("custom")` never throws. It cannot talk to any * server: real requests must go through `resolveCustomEndpoint(userId, modelKey)` which builds a * per-endpoint adapter. Every method fails loudly with an actionable message. */ export const customPlaceholderAdapter: AIProviderAdapter = { id: "custom", name: "Custom endpoint", keyDocsUrl: "/app/settings/endpoints", async validateApiKey() { return { ok: false, latencyMs: 0, error: unresolved().toJSON() }; }, async listModels() { return []; }, async chat() { throw unresolved(); }, async *streamChat() { yield { type: "error", error: unresolved().toJSON() }; }, normalizeError(error: unknown) { return error instanceof PolyProviderError ? error : unresolved(); }, }; function unresolved(): PolyProviderError { return new PolyProviderError({ code: "MODEL_NOT_FOUND", message: "Custom endpoint not resolved. Route this request through resolveCustomEndpoint().", provider: "custom", retryable: false }); }