spb/polyllm

1import { MODELS, rawPost, save, short, sleep, WEATHER_TOOL } from "./lib.ts";23const base = (model: string) => ({ model, messages: [{ role: "user", content: "Reply with the single word: ok" }], max_tokens: 60 });45const variants: Array<[string, Record<string, unknown>]> = [6  ["temperature=0", { temperature: 0 }],7  ["temperature=0.5", { temperature: 0.5 }],8  ["temperature=1", { temperature: 1 }],9  ["temperature=1.5", { temperature: 1.5 }],10  ["temperature=2", { temperature: 2 }],11  ["top_p=0.5", { top_p: 0.5 }],12  ["top_p=0.95", { top_p: 0.95 }],13  ["max_completion_tokens=60 (no max_tokens)", { max_tokens: undefined, max_completion_tokens: 60 }],14  ["stop=[\\n]", { stop: ["\n"] }],15  ["stop=6 items", { stop: ["a", "b", "c", "d", "e", "f"] }],16  ["frequency_penalty=0.5", { frequency_penalty: 0.5 }],17  ["frequency_penalty=0", { frequency_penalty: 0 }],18  ["presence_penalty=0.5", { presence_penalty: 0.5 }],19  ["presence_penalty=0", { presence_penalty: 0 }],20  ["n=2", { n: 2 }],21  ["n=1", { n: 1 }],22  ["seed=42", { seed: 42 }],23  ["logprobs=true,top_logprobs=2", { logprobs: true, top_logprobs: 2 }],24  ["thinking=enabled", { thinking: { type: "enabled" } }],25  ["thinking=disabled", { thinking: { type: "disabled" } }],26  ["thinking=enabled,keep=all", { thinking: { type: "enabled", keep: "all" } }],27  ["reasoning_effort=low", { reasoning_effort: "low" }],28  ["reasoning_effort=high", { reasoning_effort: "high" }],29  ["reasoning_effort=max", { reasoning_effort: "max" }],30  ["reasoning_effort=medium", { reasoning_effort: "medium" }],31  ["reasoning_effort=none", { reasoning_effort: "none" }],32  ["response_format=json_object", { response_format: { type: "json_object" }, messages: [{ role: "user", content: "Return a JSON object with key ok set to true." }] }],33  ["response_format=json_schema strict", { response_format: { type: "json_schema", json_schema: { name: "ok", strict: true, schema: { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"], additionalProperties: false } } }, messages: [{ role: "user", content: "Return ok=true as JSON." }] }],34  ["tools+tool_choice=auto", { tools: [WEATHER_TOOL], tool_choice: "auto" }],35  ["tools+tool_choice=none", { tools: [WEATHER_TOOL], tool_choice: "none" }],36  ["tools+tool_choice=required", { tools: [WEATHER_TOOL], tool_choice: "required", messages: [{ role: "user", content: "Weather in Montreal?" }] }],37  ["tools+tool_choice=function", { tools: [WEATHER_TOOL], tool_choice: { type: "function", function: { name: "get_weather" } }, messages: [{ role: "user", content: "Weather in Montreal?" }] }],38  ["tools+parallel_tool_calls=false", { tools: [WEATHER_TOOL], parallel_tool_calls: false }],39  ["tools+parallel_tool_calls=true", { tools: [WEATHER_TOOL], parallel_tool_calls: true }],40  ["unknown param foo_bar", { foo_bar: 1 }],41  ["developer role", { messages: [{ role: "developer", content: "Be terse." }, { role: "user", content: "Reply with the single word: ok" }] }],42  ["max_tokens=1 (truncation)", { max_tokens: 1 }],43];4445import { existsSync, readFileSync } from "node:fs";46const prevPath = new URL("./out/02-params.json", import.meta.url);47const prev: Record<string, any[]> = existsSync(prevPath) ? JSON.parse(readFileSync(prevPath, "utf8")) : {};48const onlyModels = process.argv.slice(2);49const all: Record<string, unknown> = { ...prev };50for (const model of MODELS) {51  if (onlyModels.length && !onlyModels.includes(model)) continue;52  const prevRows: any[] = prev[model] ?? [];53  const rows: any[] = [];54  for (const [label, patch] of variants) {55    const old = prevRows.find((r) => r.label === label);56    if (old && old.status === 200 || old && old.status === 400) { rows.push(old); continue; } // keep definitive results, redo 429s57    await sleep(700);58    const body: Record<string, unknown> = { ...base(model), ...patch };59    for (const k of Object.keys(body)) if (body[k] === undefined) delete body[k];60    let r: any;61    try { r = await rawPost("/chat/completions", body); } catch (e: any) { r = { status: "EXC", body: String(e) }; }62    const ch = r.body?.choices?.[0];63    const row = {64      label, status: r.status,65      finish: ch?.finish_reason ?? null,66      content: short(ch?.message?.content, 80),67      hasReasoning: typeof ch?.message?.reasoning_content === "string" && ch.message.reasoning_content.length > 0,68      toolCalls: ch?.message?.tool_calls?.length ?? 0,69      usage: r.body?.usage ?? null,70      error: r.status === 200 ? null : r.body?.error ?? r.body,71      nChoices: r.body?.choices?.length ?? null,72      logprobs: ch?.logprobs ? "present" : null,73    };74    rows.push(row);75    console.log(`[${model}] ${label.padEnd(42)} ${r.status} finish=${row.finish} reasoning=${row.hasReasoning} tc=${row.toolCalls} ${r.status !== 200 ? short(row.error, 220) : short(row.content, 40)}`);76  }77  all[model] = rows;78  save("02-params", all);79}80save("02-params", all);81