spb/polyllm

1import { chat, rawSSE, save, MODELS, short } from "./lib.ts";23const out: Record<string, unknown> = {};4// (1) json_object + stream (docs: incompatible)5for (const model of MODELS) {6  const s = await rawSSE("/chat/completions", {7    model,8    messages: [{ role: "user", content: "Return a JSON object with key answer = 4." }],9    max_completion_tokens: 150,10    response_format: { type: "json_object" },11    stream: true,12    reasoning_effort: model === "gpt-oss-120b" ? "low" : "none",13  });14  let content = "";15  for (const e of s.events) if (e.data !== "[DONE]" && e.data?.choices?.[0]?.delta?.content) content += e.data.choices[0].delta.content;16  out[`${model}:json_object_stream`] = { status: s.status, error: s.error, nEvents: s.events.length, content };17  console.log(model, "json_object+stream:", s.status, s.error ? short(s.error, 300) : `events=${s.events.length} content=${JSON.stringify(content)}`);18}19// (2) qwen over-context (retry after TPM window)20{21  const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"];22  const parts: string[] = [];23  for (let i = 0; i < 140_000; i++) parts.push(words[i % words.length]);24  const r = await chat({ model: "qwen-3.8-27b", messages: [{ role: "user", content: `Reply OK.\n${parts.join(" ")}` }], max_completion_tokens: 10, reasoning_effort: "none" });25  out["qwen:ctx140k"] = { status: r.status, body: r.body, headers: r.headers };26  console.log("qwen ctx140k:", r.status, short(r.body, 300), r.status === 429 ? JSON.stringify(r.headers) : "");27}28// (3) logprobs + reasoning_logprobs on reasoning models29for (const model of ["qwen-3.8-27b", "gpt-oss-120b"]) {30  const r = await chat({ model, messages: [{ role: "user", content: "Reply with the single word: pong" }], max_completion_tokens: 150, logprobs: true, top_logprobs: 2, reasoning_effort: "low" });31  const b: any = r.body;32  const ch = b?.choices?.[0];33  out[`${model}:logprobs`] = { status: r.status, choiceKeys: ch && Object.keys(ch), logprobs: short(ch?.logprobs, 300), reasoning_logprobs: short(ch?.reasoning_logprobs, 300), finish: ch?.finish_reason };34  console.log(model, "logprobs:", r.status, "choice keys", ch && Object.keys(ch), "\n  logprobs:", short(ch?.logprobs, 200), "\n  reasoning_logprobs:", short(ch?.reasoning_logprobs, 200));35}36// (4) qwen with `stop` and reasoning none to see stop works on content37{38  const r = await chat({ model: "qwen-3.8-27b", messages: [{ role: "user", content: "Count from 1 to 10 separated by commas." }], max_completion_tokens: 60, stop: [", 5"], reasoning_effort: "none" });39  const b: any = r.body;40  out["qwen:stop"] = { status: r.status, content: b?.choices?.[0]?.message?.content, finish: b?.choices?.[0]?.finish_reason };41  console.log("qwen stop:", r.status, JSON.stringify(b?.choices?.[0]?.message?.content), b?.choices?.[0]?.finish_reason);42}43// (5) seed determinism (gemma, 2 calls)44{45  const mk = () => chat({ model: "gemma-4-31b", messages: [{ role: "user", content: "Give me a random 6-digit number, digits only." }], max_completion_tokens: 20, seed: 7, temperature: 1 });46  const a: any = (await mk()).body, b: any = (await mk()).body;47  out["gemma:seed"] = { a: a?.choices?.[0]?.message?.content, b: b?.choices?.[0]?.message?.content, fp: [a?.system_fingerprint, b?.system_fingerprint] };48  console.log("gemma seed=7 twice:", JSON.stringify(out["gemma:seed"]));49}50// (6) prompt caching: repeat same long-ish prefix twice51{52  const prefix = "You are a helpful assistant. " + "Context paragraph about Montreal geography and history. ".repeat(40);53  const mk = () => chat({ model: "gpt-oss-120b", messages: [{ role: "system", content: prefix }, { role: "user", content: "Reply OK." }], max_completion_tokens: 20, reasoning_effort: "low" });54  const a: any = (await mk()).body, b: any = (await mk()).body;55  out["gptoss:cache"] = { first: a?.usage, second: b?.usage };56  console.log("cache: first cached", a?.usage?.prompt_tokens_details?.cached_tokens, "second cached", b?.usage?.prompt_tokens_details?.cached_tokens, "of", b?.usage?.prompt_tokens);57}58// (7) OpenAI SDK compatibility (openai@7) streaming with reasoning field59{60  const { openai } = await import("./lib.ts");61  const stream = await openai.chat.completions.create({ model: "gpt-oss-120b", messages: [{ role: "user", content: "Say hi." }], stream: true, max_completion_tokens: 100, reasoning_effort: "low" } as any);62  let reasoning = "", content = "", usage: unknown;63  for await (const chunk of stream as any) {64    const d = chunk.choices?.[0]?.delta ?? {};65    if (d.reasoning) reasoning += d.reasoning;66    if (d.content) content += d.content;67    if (chunk.usage) usage = chunk.usage;68  }69  out["openai-sdk-stream"] = { reasoning: short(reasoning, 100), content, usage };70  console.log("openai sdk stream ok: reasoning?", reasoning.length > 0, "content", JSON.stringify(content), "usage?", !!usage);71}72save("08-misc.json", out);73