spb/polyllm

1import { CHAT_MODELS, post, save, short } from "./lib.ts";23const TOOL = {4  type: "function",5  function: {6    name: "get_weather",7    description: "Get weather for a city",8    parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"] },9  },10};1112const variants: Record<string, any> = {13  base: {},14  temperature_0: { temperature: 0 },15  temperature_1: { temperature: 1.0 },16  temperature_1_5: { temperature: 1.5 },17  temperature_2: { temperature: 2.0 },18  top_p_0_5: { top_p: 0.5 },19  top_k_5: { top_k: 5 },20  max_tokens_5: { max_tokens: 5 },21  max_completion_tokens_5: { max_completion_tokens: 5 },22  stop_array: { stop: ["."] },23  stop_string: { stop: "." },24  random_seed_42: { random_seed: 42 },25  seed_42: { seed: 42 },26  presence_penalty_1: { presence_penalty: 1.0 },27  presence_penalty_2_5: { presence_penalty: 2.5 },28  frequency_penalty_1: { frequency_penalty: 1.0 },29  frequency_penalty_neg_2_5: { frequency_penalty: -2.5 },30  n_2: { n: 2 },31  safe_prompt: { safe_prompt: true },32  prompt_mode_reasoning: { prompt_mode: "reasoning" },33  reasoning_effort_none: { reasoning_effort: "none" },34  reasoning_effort_minimal: { reasoning_effort: "minimal" },35  reasoning_effort_low: { reasoning_effort: "low" },36  reasoning_effort_medium: { reasoning_effort: "medium" },37  reasoning_effort_high: { reasoning_effort: "high" },38  reasoning_effort_xhigh: { reasoning_effort: "xhigh" },39  reasoning_effort_bogus: { reasoning_effort: "bogus" },40  response_format_json_object: {41    response_format: { type: "json_object" },42    messages: [{ role: "user", content: "Return a JSON object with key ok set to true." }],43  },44  response_format_json_schema: {45    response_format: {46      type: "json_schema",47      json_schema: {48        name: "ok",49        strict: true,50        schema: { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"], additionalProperties: false },51      },52    },53  },54  parallel_tool_calls_false: { tools: [TOOL], parallel_tool_calls: false },55  tool_choice_required: { tools: [TOOL], tool_choice: "required", messages: [{ role: "user", content: "Weather in Paris?" }] },56  tool_choice_any: { tools: [TOOL], tool_choice: "any", messages: [{ role: "user", content: "Weather in Paris?" }] },57  tool_choice_object: {58    tools: [TOOL],59    tool_choice: { type: "function", function: { name: "get_weather" } },60    messages: [{ role: "user", content: "Weather in Paris?" }],61  },62  logprobs: { logprobs: true },63  developer_role: { messages: [{ role: "developer", content: "Be terse." }, { role: "user", content: "Say OK." }] },64  system_role: { messages: [{ role: "system", content: "Be terse." }, { role: "user", content: "Say OK." }] },65  unknown_param: { foo_bar: 1 },66  prediction: { prediction: { type: "content", content: "OK" } },67  service_tier_auto: { service_tier: "auto" },68  service_tier_standard_only: { service_tier: "standard_only" },69  prompt_cache_key: { prompt_cache_key: "polyllm-probe" },70  stream_options_nonstream: { stream_options: { include_usage: true } },71  assistant_prefix: {72    messages: [73      { role: "user", content: "Say OK." },74      { role: "assistant", content: "Sure thing:", prefix: true },75    ],76  },77  metadata: { metadata: { app: "polyllm" } },78};7980async function pool<T>(items: T[], n: number, fn: (t: T) => Promise<void>) {81  let i = 0;82  await Promise.all(83    Array.from({ length: n }, async () => {84      while (i < items.length) await fn(items[i++]);85    }),86  );87}8889const results: Record<string, Record<string, any>> = {};90for (const model of CHAT_MODELS) {91  results[model] = {};92  console.log(`\n=== ${model}`);93  await pool(Object.entries(variants), 5, async ([name, v]) => {94    const payload = { model, messages: [{ role: "user", content: "Say OK." }], max_tokens: 30, ...v };95    const r = await post("/chat/completions", payload);96    const body: any = r.body;97    let info = "";98    if (r.status === 200) {99      const ch = body.choices?.[0];100      const c = ch?.message?.content;101      const shape = Array.isArray(c) ? "array:" + c.map((x: any) => x.type).join(",") : typeof c;102      info = `finish=${ch?.finish_reason} choices=${body.choices?.length} content=${shape} ${short(c, 80)} tool_calls=${ch?.message?.tool_calls ? ch.message.tool_calls.length : 0} usage=${JSON.stringify(body.usage)}${ch?.logprobs ? " logprobs=" + short(ch.logprobs, 80) : ""}`;103    } else {104      info = short(body, 400);105    }106    results[model][name] = { status: r.status, body: r.status === 200 ? { finish: body.choices?.[0]?.finish_reason, content: body.choices?.[0]?.message?.content, usage: body.usage, choices: body.choices?.length } : body };107    console.log(`${name.padEnd(30)} ${r.status} ${info}`);108  });109}110save("02-params.json", results);111