spb/polyllm

1import { chat, rawSSE, save, MODELS, short } from "./lib.ts";23const schema = {4  type: "object",5  properties: {6    city: { type: "string" },7    country: { type: "string" },8    population_millions: { type: "number" },9    landmarks: { type: "array", items: { type: "string" } },10  },11  required: ["city", "country", "population_millions", "landmarks"],12  additionalProperties: false,13};1415const out: Record<string, unknown> = {};16for (const model of MODELS) {17  const payload = {18    model,19    messages: [{ role: "user", content: "Describe Montreal. Two landmarks max." }],20    max_completion_tokens: 200,21    response_format: { type: "json_schema", json_schema: { name: "city_info", strict: true, schema } },22  };23  const r = await chat(payload);24  const b: any = r.body;25  let parsed: unknown = null;26  let valid = false;27  try {28    parsed = JSON.parse(b?.choices?.[0]?.message?.content ?? "");29    valid = typeof (parsed as any).city === "string" && Array.isArray((parsed as any).landmarks);30  } catch {}31  out[`${model}:json_schema`] = { status: r.status, valid, parsed, finish: b?.choices?.[0]?.finish_reason, usage: b?.usage, reasoning: short(b?.choices?.[0]?.message?.reasoning, 100), error: r.status !== 200 ? b : undefined };32  console.log(model, "json_schema strict:", r.status, "valid", valid, short(parsed ?? b, 250));3334  // streamed json_schema35  const s = await rawSSE("/chat/completions", { ...payload, stream: true });36  let content = "";37  for (const e of s.events) if (e.data !== "[DONE]" && e.data?.choices?.[0]?.delta?.content) content += e.data.choices[0].delta.content;38  let sValid = false;39  try {40    sValid = typeof JSON.parse(content).city === "string";41  } catch {}42  out[`${model}:json_schema_stream`] = { status: s.status, valid: sValid, content: short(content, 300), error: s.error };43  console.log(model, "json_schema stream:", s.status, "valid", sValid);4445  // schema with a forbidden keyword (pattern) to capture the error shape46  const r3 = await chat({47    ...payload,48    response_format: { type: "json_schema", json_schema: { name: "x", strict: true, schema: { type: "object", properties: { code: { type: "string", pattern: "^[A-Z]{3}$" } }, required: ["code"], additionalProperties: false } } },49  });50  out[`${model}:json_schema_pattern`] = { status: r3.status, body: r3.status !== 200 ? r3.body : short((r3.body as any)?.choices?.[0]?.message?.content, 100) };51  console.log(model, "json_schema with pattern:", r3.status, short(r3.body, 300));5253  // strict without additionalProperties:false54  const r4 = await chat({55    ...payload,56    response_format: { type: "json_schema", json_schema: { name: "x", strict: true, schema: { type: "object", properties: { a: { type: "string" } }, required: ["a"] } } },57  });58  out[`${model}:json_schema_no_addprops`] = { status: r4.status, body: r4.status !== 200 ? r4.body : short((r4.body as any)?.choices?.[0]?.message?.content, 100) };59  console.log(model, "strict w/o additionalProperties:false:", r4.status, short(r4.body, 300));60}61save("04-structured.json", out);62