spb/polyllm

1import { chat, save, MODELS, short } from "./lib.ts";23const base = (model: string) => ({4  model,5  messages: [{ role: "user", content: "Reply with the single word: pong" }],6  max_completion_tokens: 40,7});89const tool = {10  type: "function",11  function: {12    name: "get_weather",13    description: "Get weather for a city",14    parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"], additionalProperties: false },15  },16};1718const variants: Record<string, (m: string) => Record<string, unknown>> = {19  temperature_0_5: (m) => ({ ...base(m), temperature: 0.5 }),20  temperature_2: (m) => ({ ...base(m), temperature: 2 }),21  temperature_2_5: (m) => ({ ...base(m), temperature: 2.5 }),22  top_p: (m) => ({ ...base(m), top_p: 0.9 }),23  top_k: (m) => ({ ...base(m), top_k: 40 }),24  max_tokens: (m) => ({ model: m, messages: base(m).messages, max_tokens: 40 }),25  both_max: (m) => ({ ...base(m), max_tokens: 40 }),26  min_tokens: (m) => ({ ...base(m), min_completion_tokens: 5 }),27  stop: (m) => ({ ...base(m), stop: ["\n", "END"] }),28  seed: (m) => ({ ...base(m), seed: 42 }),29  frequency_penalty: (m) => ({ ...base(m), frequency_penalty: 0.5 }),30  presence_penalty: (m) => ({ ...base(m), presence_penalty: 0.5 }),31  logit_bias: (m) => ({ ...base(m), logit_bias: { "1234": 5 } }),32  logprobs: (m) => ({ ...base(m), logprobs: true, top_logprobs: 2 }),33  n_2: (m) => ({ ...base(m), n: 2 }),34  reasoning_effort_none: (m) => ({ ...base(m), reasoning_effort: "none" }),35  reasoning_effort_low: (m) => ({ ...base(m), reasoning_effort: "low" }),36  reasoning_effort_medium: (m) => ({ ...base(m), reasoning_effort: "medium" }),37  reasoning_effort_high: (m) => ({ ...base(m), reasoning_effort: "high" }),38  reasoning_effort_minimal: (m) => ({ ...base(m), reasoning_effort: "minimal" }),39  disable_reasoning: (m) => ({ ...base(m), disable_reasoning: true }),40  reasoning_format_parsed: (m) => ({ ...base(m), reasoning_format: "parsed" }),41  reasoning_format_raw: (m) => ({ ...base(m), reasoning_format: "raw" }),42  reasoning_format_hidden: (m) => ({ ...base(m), reasoning_format: "hidden" }),43  clear_thinking: (m) => ({ ...base(m), clear_thinking: true }),44  json_object: (m) => ({45    model: m,46    messages: [{ role: "user", content: "Return a JSON object with key answer = 4." }],47    max_completion_tokens: 60,48    response_format: { type: "json_object" },49  }),50  json_object_stream: (m) => ({51    model: m,52    messages: [{ role: "user", content: "Return a JSON object with key answer = 4." }],53    max_completion_tokens: 60,54    response_format: { type: "json_object" },55    stream: true,56  }),57  json_schema_strict: (m) => ({58    model: m,59    messages: [{ role: "user", content: "Give the sum of 2 and 2." }],60    max_completion_tokens: 60,61    response_format: {62      type: "json_schema",63      json_schema: {64        name: "sum",65        strict: true,66        schema: { type: "object", properties: { answer: { type: "integer" } }, required: ["answer"], additionalProperties: false },67      },68    },69  }),70  tools_auto: (m) => ({ ...base(m), messages: [{ role: "user", content: "Weather in Montreal?" }], max_completion_tokens: 100, tools: [tool], tool_choice: "auto" }),71  tools_required: (m) => ({ ...base(m), messages: [{ role: "user", content: "Weather in Montreal?" }], max_completion_tokens: 100, tools: [tool], tool_choice: "required" }),72  tools_named: (m) => ({ ...base(m), messages: [{ role: "user", content: "Weather in Montreal?" }], max_completion_tokens: 100, tools: [tool], tool_choice: { type: "function", function: { name: "get_weather" } } }),73  tools_parallel_false: (m) => ({ ...base(m), messages: [{ role: "user", content: "Weather in Montreal and Paris?" }], max_completion_tokens: 100, tools: [tool], parallel_tool_calls: false }),74  tools_strict: (m) => ({ ...base(m), messages: [{ role: "user", content: "Weather in Montreal?" }], max_completion_tokens: 100, tools: [{ ...tool, function: { ...tool.function, strict: true } }] }),75  tools_plus_json_schema: (m) => ({76    ...base(m),77    messages: [{ role: "user", content: "Weather in Montreal?" }],78    max_completion_tokens: 100,79    tools: [tool],80    response_format: { type: "json_schema", json_schema: { name: "x", strict: true, schema: { type: "object", properties: { a: { type: "string" } }, required: ["a"], additionalProperties: false } } },81  }),82  developer_role: (m) => ({ ...base(m), messages: [{ role: "developer", content: "Be terse." }, ...base(m).messages] }),83  system_role: (m) => ({ ...base(m), messages: [{ role: "system", content: "Be terse." }, ...base(m).messages] }),84  unknown_param: (m) => ({ ...base(m), foo_bar: 1 }),85  user_field: (m) => ({ ...base(m), user: "probe-user" }),86  service_tier_priority: (m) => ({ ...base(m), service_tier: "priority" }),87  prompt_cache_key: (m) => ({ ...base(m), prompt_cache_key: "polyllm-probe" }),88  logprobs_alone: (m) => ({ ...base(m), logprobs: true }),89  max_completion_tokens_5: (m) => ({ ...base(m), messages: [{ role: "user", content: "Write 3 sentences about Montreal." }], max_completion_tokens: 5 }),90};9192const out: Record<string, Record<string, unknown>> = {};93for (const model of MODELS) {94  out[model] = {};95  for (const [name, mk] of Object.entries(variants)) {96    const payload = mk(model);97    let r: any;98    if (payload.stream) {99      // streaming variant: just record status + first event/body100      const res = await fetch("https://api.cerebras.ai/v1/chat/completions", {101        method: "POST",102        headers: { "Content-Type": "application/json", Authorization: `Bearer ${process.env.CEREBRAS_API_KEY}` },103        body: JSON.stringify(payload),104      });105      const text = await res.text();106      let body: unknown = text;107      try {108        body = JSON.parse(text);109      } catch {}110      r = { status: res.status, body: typeof body === "string" ? short(body, 500) : body };111    } else {112      r = await chat(payload);113    }114    const b: any = r.body;115    const msg = b?.choices?.[0]?.message;116    const summary = {117      status: r.status,118      finish: b?.choices?.[0]?.finish_reason,119      content: short(msg?.content, 160),120      reasoning: msg?.reasoning !== undefined ? short(msg.reasoning, 100) : undefined,121      reasoningLen: typeof msg?.reasoning === "string" ? msg.reasoning.length : undefined,122      tool_calls: msg?.tool_calls ? short(msg.tool_calls, 200) : undefined,123      usage: b?.usage,124      logprobs: b?.choices?.[0]?.logprobs ? short(b.choices[0].logprobs, 200) : undefined,125      nChoices: b?.choices?.length,126      error: r.status !== 200 ? b : undefined,127      system_fingerprint: b?.system_fingerprint,128      service_tier: b?.service_tier,129      service_tier_used: b?.service_tier_used,130    };131    out[model][name] = summary;132    console.log(`${model} | ${name} | ${r.status} | ${summary.finish ?? ""} | ${short(summary.error ?? summary.content, 200)}${summary.reasoningLen !== undefined ? ` | reasoningLen=${summary.reasoningLen} rt=${b?.usage?.completion_tokens_details?.reasoning_tokens}` : ""}`);133  }134}135save("02-params.json", out);136