spb/polyllm

1import { chat, rawSSE, save, MODELS, short } from "./lib.ts";23const out: Record<string, unknown> = {};4for (const model of MODELS) {5  // (a) tiny non-streaming6  const r = await chat({7    model,8    messages: [{ role: "user", content: "What is 2+2? Answer in one short sentence." }],9    max_completion_tokens: 200,10  });11  const b: any = r.body;12  const usage = b?.usage;13  const tps = usage && b?.time_info ? Math.round(usage.completion_tokens / b.time_info.completion_time) : null;14  out[`${model}:nonstream`] = { ...r, tokensPerSecond: tps };15  console.log(model, "nonstream", r.status, r.ms + "ms", "tps=", tps, "\n  message keys:", b?.choices?.[0]?.message && Object.keys(b.choices[0].message), "\n  content:", short(b?.choices?.[0]?.message?.content, 200), "\n  reasoning:", short(b?.choices?.[0]?.message?.reasoning, 120), "\n  usage:", JSON.stringify(usage), "\n  time_info:", JSON.stringify(b?.time_info), "\n  top keys:", b && Object.keys(b));1617  // (b) streaming with include_usage18  const s = await rawSSE("/chat/completions", {19    model,20    messages: [{ role: "user", content: "Say hello in French, 5 words max." }],21    max_completion_tokens: 200,22    stream: true,23    stream_options: { include_usage: true },24  });25  const deltaKeys = new Set<string>();26  const finishes: string[] = [];27  let content = "";28  let reasoning = "";29  const withUsage: number[] = [];30  const withTimeInfo: number[] = [];31  const otherKeys = new Set<string>();32  s.events.forEach((e, i) => {33    if (e.data === "[DONE]") return;34    const d = e.data;35    Object.keys(d).forEach((k) => otherKeys.add(k));36    if (d.usage) withUsage.push(i);37    if (d.time_info) withTimeInfo.push(i);38    const c = d.choices?.[0];39    if (c?.delta) Object.keys(c.delta).forEach((k) => deltaKeys.add(k));40    if (c?.delta?.content) content += c.delta.content;41    if (c?.delta?.reasoning) reasoning += c.delta.reasoning;42    if (c?.delta?.reasoning_content) reasoning += "[RC]" + c.delta.reasoning_content;43    if (c?.finish_reason) finishes.push(`${i}:${c.finish_reason}`);44  });45  out[`${model}:stream`] = {46    status: s.status,47    headers: s.headers,48    ttfbMs: s.ttfbMs,49    totalMs: s.totalMs,50    nEvents: s.events.length,51    deltaKeys: [...deltaKeys],52    topLevelKeys: [...otherKeys],53    finishes,54    withUsage,55    withTimeInfo,56    content,57    reasoning: short(reasoning, 400),58    hasThinkTag: /<think>/.test(content),59    first3: s.events.slice(0, 3),60    last3: s.events.slice(-3),61    error: s.error,62  };63  console.log(model, "stream", s.status, "events", s.events.length, "ttfb", s.ttfbMs, "deltaKeys", [...deltaKeys], "finishes", finishes, "usageAt", withUsage, "timeInfoAt", withTimeInfo, "\n  content:", short(content, 150), "\n  reasoning:", short(reasoning, 150), "\n  last3:", short(s.events.slice(-3), 900));64  // (b2) streaming WITHOUT stream_options: is usage still present?65  const s2 = await rawSSE("/chat/completions", {66    model,67    messages: [{ role: "user", content: "Say hi." }],68    max_completion_tokens: 30,69    stream: true,70  });71  const usageIdx = s2.events.map((e, i) => (e.data !== "[DONE]" && e.data?.usage ? i : -1)).filter((i) => i >= 0);72  out[`${model}:stream-no-options`] = { status: s2.status, nEvents: s2.events.length, usageIdx, last2: s2.events.slice(-2) };73  console.log(model, "stream w/o stream_options: usage at", usageIdx, "of", s2.events.length);74}75save("01-chat-stream.json", out);76