import { chat, rawSSE, save, MODELS, short } from "./lib.ts"; const out: Record = {}; // (1) gpt-oss json_schema streamed: inspect chunks (why invalid in 04?) { const s = await rawSSE("/chat/completions", { model: "gpt-oss-120b", messages: [{ role: "user", content: "Give the sum of 2 and 2." }], max_completion_tokens: 200, stream: true, response_format: { type: "json_schema", json_schema: { name: "sum", strict: true, schema: { type: "object", properties: { answer: { type: "integer" } }, required: ["answer"], additionalProperties: false } } }, }); let content = ""; let reasoning = ""; for (const e of s.events) { if (e.data === "[DONE]") continue; const d = e.data.choices?.[0]?.delta ?? {}; if (d.content) content += d.content; if (d.reasoning) reasoning += d.reasoning; } const fin = s.events.find((e) => e.data !== "[DONE]" && e.data?.choices?.[0]?.finish_reason)?.data?.choices?.[0]?.finish_reason; out.gptOssJsonSchemaStream = { status: s.status, content, reasoning: short(reasoning, 300), finish: fin, usage: s.events.find((e) => e.data !== "[DONE]" && e.data?.usage)?.data?.usage }; console.log("gpt-oss json_schema stream:", s.status, "finish", fin, "content:", JSON.stringify(content), "reasoning:", short(reasoning, 150)); } // (2) reasoning_format raw on gpt-oss and qwen: how are /analysis tags embedded? (non-stream, low effort) for (const model of ["gpt-oss-120b", "qwen-3.8-27b", "gemma-4-31b"]) { const r = await chat({ model, messages: [{ role: "user", content: "Is 7 prime? One word." }], max_completion_tokens: 300, reasoning_format: "raw", reasoning_effort: "low" }); const b: any = r.body; out[`${model}:raw`] = { status: r.status, message: b?.choices?.[0]?.message, usage: b?.usage, error: r.status !== 200 ? b : undefined }; console.log(model, "reasoning_format raw:", r.status, short(b?.choices?.[0]?.message ?? b, 500)); } // (3) gemma with reasoning enabled: is `reasoning` returned & streamed? { const s = await rawSSE("/chat/completions", { model: "gemma-4-31b", messages: [{ role: "user", content: "Is 7 prime? One word." }], max_completion_tokens: 300, stream: true, reasoning_effort: "medium" }); const keys = new Set(); let reasoning = ""; let content = ""; for (const e of s.events) { if (e.data === "[DONE]") continue; const d = e.data.choices?.[0]?.delta ?? {}; Object.keys(d).forEach((k) => keys.add(k)); if (d.reasoning) reasoning += d.reasoning; if (d.content) content += d.content; } out.gemmaReasoningStream = { status: s.status, deltaKeys: [...keys], reasoning: short(reasoning, 300), content, usage: s.events.find((e) => e.data !== "[DONE]" && e.data?.usage)?.data?.usage, error: s.error }; console.log("gemma reasoning stream:", s.status, [...keys], "reasoning:", short(reasoning, 150), "content:", content, "usage:", JSON.stringify((out.gemmaReasoningStream as any).usage)); } // (4) Context window: ~70k-token prompt (paid: 128k/131k; free: 64k/65k) then ~140k (over limit) to capture error text. function filler(tokensApprox: number) { // "lorem ipsum " ~ 3 tokens per 12 chars → build ~4 chars/token filler of distinct words to avoid weirdness const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"]; const parts: string[] = []; for (let i = 0; i < tokensApprox; i++) parts.push(words[i % words.length]); return parts.join(" "); } for (const model of MODELS) { for (const target of [70_000, 140_000]) { const r = await chat({ model, messages: [{ role: "user", content: `Reply with the single word OK. Ignore this filler:\n${filler(target)}` }], max_completion_tokens: 10, reasoning_effort: model === "gpt-oss-120b" ? "low" : "none", }); const b: any = r.body; out[`${model}:ctx${target}`] = { status: r.status, promptTokens: b?.usage?.prompt_tokens, finish: b?.choices?.[0]?.finish_reason, error: r.status !== 200 ? b : undefined, headers: r.headers, ms: r.ms }; console.log(model, `ctx~${target}:`, r.status, "prompt_tokens", b?.usage?.prompt_tokens, r.ms + "ms", r.status !== 200 ? short(b, 400) : ""); } } // (5) max_completion_tokens above the per-model cap → error text? for (const model of MODELS) { const r = await chat({ model, messages: [{ role: "user", content: "hi" }], max_completion_tokens: 60_000, reasoning_effort: model === "gpt-oss-120b" ? "low" : "none" }); out[`${model}:maxout60k`] = { status: r.status, body: r.status !== 200 ? r.body : short((r.body as any)?.choices?.[0]?.message?.content, 80) }; console.log(model, "max_completion_tokens 60000:", r.status, short(r.body, 300)); } // (6) reasoning-heavy probe: gpt-oss high effort, see reasoning_tokens vs completion_tokens & finish on cap { const r = await chat({ model: "gpt-oss-120b", messages: [{ role: "user", content: "How many r's in 'strawberry'? Think carefully." }], max_completion_tokens: 1000, reasoning_effort: "high" }); const b: any = r.body; out.gptOssHigh = { status: r.status, usage: b?.usage, time_info: b?.time_info, finish: b?.choices?.[0]?.finish_reason, content: short(b?.choices?.[0]?.message?.content, 200), reasoningLen: b?.choices?.[0]?.message?.reasoning?.length }; console.log("gpt-oss high:", r.status, JSON.stringify(b?.usage), "finish", b?.choices?.[0]?.finish_reason, "tps", Math.round(b?.usage?.completion_tokens / b?.time_info?.completion_time)); } // (7) reasoning cap: qwen with tiny max_completion_tokens while reasoning on → finish_reason length, content empty? { const r = await chat({ model: "qwen-3.8-27b", messages: [{ role: "user", content: "Explain gravity." }], max_completion_tokens: 20 }); const b: any = r.body; out.qwenCap20 = { status: r.status, message: b?.choices?.[0]?.message, finish: b?.choices?.[0]?.finish_reason, usage: b?.usage }; console.log("qwen cap 20:", r.status, "finish", b?.choices?.[0]?.finish_reason, JSON.stringify(b?.choices?.[0]?.message), JSON.stringify(b?.usage)); } save("07-context-reasoning.json", out);