spb/polyllm

1// Probe (c): which parameters are accepted vs rejected per model (exact error bodies).2import { CHAT_MODELS, raw, save, short } from "./lib.ts";34const base = { messages: [{ role: "user", content: "Reply with the single word: ok" }] };5const variants: Record<string, any> = {6  temperature: { temperature: 0.3, max_completion_tokens: 50 },7  top_p: { top_p: 0.9, max_completion_tokens: 50 },8  frequency_penalty: { frequency_penalty: 0.5, max_completion_tokens: 50 },9  presence_penalty: { presence_penalty: 0.5, max_completion_tokens: 50 },10  seed: { seed: 42, max_completion_tokens: 50 },11  stop: { stop: ["\n"], max_completion_tokens: 50 },12  max_tokens: { max_tokens: 50 },13  max_completion_tokens: { max_completion_tokens: 50 },14  "reasoning_effort:none": { reasoning_effort: "none", max_completion_tokens: 50 },15  "reasoning_effort:low": { reasoning_effort: "low", max_completion_tokens: 200 },16  "reasoning_effort:medium": { reasoning_effort: "medium", max_completion_tokens: 200 },17  "reasoning_effort:high": { reasoning_effort: "high", max_completion_tokens: 400 },18  "reasoning_effort:xhigh": { reasoning_effort: "xhigh", max_completion_tokens: 400 },19  "reasoning_effort:minimal": { reasoning_effort: "minimal", max_completion_tokens: 50 },20  "response_format:json_object": {21    response_format: { type: "json_object" },22    messages: [{ role: "user", content: 'Return JSON {"ok":true}' }],23    max_completion_tokens: 100,24  },25  top_k: { top_k: 40, max_completion_tokens: 50 },26  logprobs: { logprobs: true, top_logprobs: 2, max_completion_tokens: 20 },27  n2: { n: 2, max_completion_tokens: 20 },28  developer_role: { messages: [{ role: "developer", content: "Be terse." }, { role: "user", content: "ok?" }], max_completion_tokens: 20 },29  unknown_param: { foo_bar: 1, max_completion_tokens: 20 },30};3132const results: Record<string, Record<string, any>> = {};33for (const model of CHAT_MODELS) {34  results[model] = {};35  const jobs = Object.entries(variants).map(async ([name, v]) => {36    const payload = { model, ...base, ...v };37    const r = await raw("/chat/completions", { method: "POST", body: JSON.stringify(payload) });38    const b: any = r.body;39    const summary =40      r.status === 20041        ? {42            status: 200,43            finish: b.choices?.[0]?.finish_reason,44            content: short(b.choices?.[0]?.message?.content, 60),45            reasoning_tokens: b.usage?.completion_tokens_details?.reasoning_tokens,46            completion_tokens: b.usage?.completion_tokens,47            n_choices: b.choices?.length,48            system_fingerprint: b.system_fingerprint,49            has_logprobs: b.choices?.[0]?.logprobs != null,50          }51        : { status: r.status, error: b };52    results[model][name] = summary;53    console.log(model.padEnd(30), name.padEnd(28), r.status, r.status === 200 ? `rt=${summary.reasoning_tokens} ct=${summary.completion_tokens} ${summary.content}` : short(b, 220));54  });55  await Promise.all(jobs);56}57save("02-params.json", results);58