spb/polyllm

1// (c) parameter acceptance matrix, one request per (model, variant). Exact error strings captured.2import { raw, MODELS, save } from "./lib.ts";34const base = (model: string) => ({ model, messages: [{ role: "user", content: "Reply with the single word: pong" }], max_tokens: 100 });5const variants: Record<string, Record<string, unknown>> = {6  "temperature=0.2": { temperature: 0.2 },7  "temperature=2": { temperature: 2 },8  "temperature=2.5": { temperature: 2.5 },9  "temperature=-1": { temperature: -1 },10  "top_p=0.5": { top_p: 0.5 },11  "top_p=1.5": { top_p: 1.5 },12  "max_tokens=1": { max_tokens: 1 },13  "max_tokens=384000": { max_tokens: 384000 },14  "max_tokens=400000": { max_tokens: 400000 },15  "max_tokens=1000000": { max_tokens: 1000000 },16  "max_completion_tokens=50": { max_completion_tokens: 50, max_tokens: undefined },17  "stop=[pong]": { stop: ["pong", "Pong"] },18  "stop=17items": { stop: Array.from({ length: 17 }, (_, i) => `zz${i}`) },19  "frequency_penalty=1": { frequency_penalty: 1 },20  "presence_penalty=1": { presence_penalty: 1 },21  "logprobs+top_logprobs=3": { logprobs: true, top_logprobs: 3 },22  "top_logprobs=25": { logprobs: true, top_logprobs: 25 },23  "thinking.disabled": { thinking: { type: "disabled" } },24  "thinking.enabled": { thinking: { type: "enabled" } },25  "thinking.bogus": { thinking: { type: "auto" } },26  "thinking.enabled+nested_effort=low": { thinking: { type: "enabled", reasoning_effort: "low" } },27  "reasoning_effort=low": { reasoning_effort: "low" },28  "reasoning_effort=medium": { reasoning_effort: "medium" },29  "reasoning_effort=high": { reasoning_effort: "high" },30  "reasoning_effort=xhigh": { reasoning_effort: "xhigh" },31  "reasoning_effort=max": { reasoning_effort: "max" },32  "reasoning_effort=none": { reasoning_effort: "none" },33  "reasoning_effort=minimal": { reasoning_effort: "minimal" },34  "reasoning_effort=low+thinking.disabled": { reasoning_effort: "low", thinking: { type: "disabled" } },35  "thinking.disabled+temperature=1.5+top_p=0.9": { thinking: { type: "disabled" }, temperature: 1.5, top_p: 0.9 },36  "response_format=json_object": { response_format: { type: "json_object" }, messages: [{ role: "user", content: "Return a json object {\"word\":\"pong\"}" }] },37  "response_format=json_schema": { response_format: { type: "json_schema", json_schema: { name: "w", schema: { type: "object", properties: { word: { type: "string" } }, required: ["word"], additionalProperties: false }, strict: true } }, messages: [{ role: "user", content: "Return json {\"word\":\"pong\"}" }] },38  "response_format=text": { response_format: { type: "text" } },39  "n=2": { n: 2 },40  "seed=42": { seed: 42 },41  "parallel_tool_calls=false": { parallel_tool_calls: false, tools: [{ type: "function", function: { name: "noop", description: "does nothing", parameters: { type: "object", properties: {} } } }], tool_choice: "none" },42  "tool_choice=none": { tools: [{ type: "function", function: { name: "noop", description: "does nothing", parameters: { type: "object", properties: {} } } }], tool_choice: "none" },43  "unknown_param": { foo_bar: 1 },44  "developer_role": { messages: [{ role: "developer", content: "Be terse." }, { role: "user", content: "Reply with the single word: pong" }] },45  "user_id": { user_id: "polyllm-probe-1" },46  "user": { user: "polyllm-probe-1" },47  "name_on_user": { messages: [{ role: "user", content: "Reply with the single word: pong", name: "simon" }] },48  "reasoning_effort_in_stream_nonstream": { stream: false, reasoning_effort: "low" },49};5051const out: Record<string, unknown> = {};52for (const model of MODELS) {53  for (const [name, extra] of Object.entries(variants)) {54    const body: Record<string, unknown> = { ...base(model), ...extra };55    for (const k of Object.keys(body)) if (body[k] === undefined) delete body[k];56    const t0 = Date.now();57    const r = await raw("/chat/completions", { method: "POST", body: JSON.stringify(body) });58    const b: any = r.body;59    const summary = r.status === 20060      ? { ok: true, ms: Date.now() - t0, finish: b.choices?.[0]?.finish_reason, nChoices: b.choices?.length, content: (b.choices?.[0]?.message?.content ?? "").slice(0, 60), reasoning_len: b.choices?.[0]?.message?.reasoning_content?.length ?? null, logprobs: b.choices?.[0]?.logprobs ? "present" : null, usage: b.usage, system_fingerprint: b.system_fingerprint }61      : { ok: false, status: r.status, body: b };62    out[`${model}|${name}`] = summary;63    console.log(model, name, "→", r.status, r.status === 200 ? `finish=${summary.finish} reasoning_tokens=${b.usage?.completion_tokens_details?.reasoning_tokens} rl=${summary.reasoning_len} "${summary.content}"` : JSON.stringify(b).slice(0, 300));64  }65}66save("02-params.json", out);67