// Probe 09: Anthropic thinking (reasoning.max_tokens) + tools, streamed, then round 2 passing reasoning_details back (and once without); // stream without any usage flag; /generation lookup with a longer delay; /models vs /models/user diff. import { readFileSync } from "node:fs"; import { raw, rawSSE, save, short, OUT } from "./lib.ts"; import { ANTHROPIC, GOOGLE, OPENAI } from "./models.ts"; const results: any = {}; const tools = [ { type: "function", function: { name: "get_weather", description: "Get the current weather for a city", parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"] } } }, ]; async function streamCollect(body: any) { const s = await rawSSE("/chat/completions", { ...body, stream: true }); const acc: Record = {}; const details: any[] = []; let reasoning = ""; let content = ""; let finish: string | undefined; let usage: any; const detailChunks: any[] = []; for (const e of s.events) { if (e.data === "[DONE]") continue; const c = e.data?.choices?.[0]; const d = c?.delta ?? {}; if (d.reasoning) reasoning += d.reasoning; if (d.content) content += d.content; if (d.reasoning_details) { details.push(...d.reasoning_details); if (detailChunks.length < 3) detailChunks.push(d.reasoning_details); } for (const tc of d.tool_calls ?? []) { const i = tc.index ?? 0; acc[i] ??= { id: tc.id, type: "function", function: { name: "", arguments: "" } }; if (tc.id) acc[i].id = tc.id; if (tc.function?.name) acc[i].function.name += tc.function.name; if (tc.function?.arguments) acc[i].function.arguments += tc.function.arguments; } if (c?.finish_reason) finish = c.finish_reason; if (e.data?.usage) usage = e.data.usage; if (e.data?.error) results.streamError = e.data; } return { status: s.status, error: (s as any).error, toolCalls: Object.values(acc), details, detailChunks, reasoning, content, finish, usage, events: s.events.length }; } for (const model of [ANTHROPIC, GOOGLE]) { const messages: any[] = [{ role: "user", content: "What's the weather in Montreal right now? Use the tool." }]; const r1 = await streamCollect({ model, messages, tools, max_tokens: 1500, reasoning: { max_tokens: 1024 } }); console.log(`\n[${model}] round1 ${r1.status} finish=${r1.finish} calls=${JSON.stringify(r1.toolCalls)} rt=${r1.usage?.completion_tokens_details?.reasoning_tokens} reasoning=${short(r1.reasoning, 80)}`); console.log(" detail types:", [...new Set(r1.details.map((d) => d.type))], "count:", r1.details.length); console.log(" first detail chunks:", short(r1.detailChunks.map((c: any[]) => c.map((d) => ({ ...d, text: d.text?.slice(0, 30), signature: d.signature?.slice(0, 16), data: d.data?.slice(0, 16) }))), 600)); if (r1.error) console.log(" ERR", short(r1.error)); results[`${model}_round1`] = r1; if (!r1.toolCalls.length) continue; // Merge reasoning_details by index/type (docs: pass back unmodified; streaming yields fragments) const merged: any[] = []; for (const d of r1.details) { const last = merged[merged.length - 1]; if (last && last.type === d.type && last.index === d.index) { if (d.text) last.text = (last.text ?? "") + d.text; if (d.summary) last.summary = (last.summary ?? "") + d.summary; if (d.data) last.data = (last.data ?? "") + d.data; if (d.signature) last.signature = d.signature; if (d.id) last.id = d.id; } else merged.push({ ...d }); } results[`${model}_merged_details`] = merged.map((d) => ({ ...d, text: d.text?.slice(0, 60), signature: d.signature?.slice(0, 24), data: d.data?.slice(0, 24) })); console.log(" merged details:", short(results[`${model}_merged_details`], 500)); const toolMsgs = r1.toolCalls.map((tc: any) => ({ role: "tool", tool_call_id: tc.id, content: JSON.stringify({ city: "Montreal", temp_c: 21, sky: "sunny" }) })); // (a) with reasoning_details passed back const withDetails = [...messages, { role: "assistant", content: r1.content || null, tool_calls: r1.toolCalls, reasoning_details: merged }, ...toolMsgs]; const a = await raw("/chat/completions", { method: "POST", body: JSON.stringify({ model, messages: withDetails, tools, max_tokens: 1500, reasoning: { max_tokens: 1024 } }) }); const ab: any = a.body; results[`${model}_round2_with_details`] = { status: a.status, finish: ab?.choices?.[0]?.finish_reason, content: ab?.choices?.[0]?.message?.content, usage: ab?.usage, error: ab?.error }; console.log(` round2 WITH details: ${a.status} ${short(ab?.choices?.[0]?.message?.content, 100)} ${ab?.error ? "ERR=" + short(ab.error, 300) : ""}`); // (b) without reasoning_details const without = [...messages, { role: "assistant", content: r1.content || null, tool_calls: r1.toolCalls }, ...toolMsgs]; const b = await raw("/chat/completions", { method: "POST", body: JSON.stringify({ model, messages: without, tools, max_tokens: 1500, reasoning: { max_tokens: 1024 } }) }); const bb: any = b.body; results[`${model}_round2_without_details`] = { status: b.status, finish: bb?.choices?.[0]?.finish_reason, content: bb?.choices?.[0]?.message?.content, usage: bb?.usage, error: bb?.error }; console.log(` round2 WITHOUT details: ${b.status} ${short(bb?.choices?.[0]?.message?.content, 100)} ${bb?.error ? "ERR=" + short(bb.error, 300) : ""}`); } // Stream with no usage flag at all { const s = await rawSSE("/chat/completions", { model: OPENAI, messages: [{ role: "user", content: "hi" }], max_tokens: 5, stream: true, reasoning: { effort: "none" } }); const usageChunk = s.events.find((e) => e.data?.usage)?.data; results.stream_no_usage_flag = { status: s.status, hasUsage: !!usageChunk, usage: usageChunk?.usage }; console.log(`\n[stream without usage flag] ${s.status} usage present=${!!usageChunk} ${JSON.stringify(usageChunk?.usage)}`); // and with stream_options.include_usage (OpenAI SDK style) const s2 = await rawSSE("/chat/completions", { model: OPENAI, messages: [{ role: "user", content: "hi" }], max_tokens: 5, stream: true, stream_options: { include_usage: true }, reasoning: { effort: "none" } }); results.stream_options_include_usage = { status: s2.status, hasUsage: s2.events.some((e) => e.data?.usage), error: (s2 as any).error }; console.log(`[stream_options.include_usage] ${s2.status} usage present=${results.stream_options_include_usage.hasUsage} ${(s2 as any).error ? JSON.stringify((s2 as any).error) : ""}`); } // /generation lookup with a longer delay { const chat = JSON.parse(readFileSync(`${OUT}01-chat.json`, "utf8")); const id = chat[OPENAI]?.body?.id; const g = await raw(`/generation?id=${id}`); results.generation = { id, status: g.status, body: g.body }; console.log(`\n[generation ${id}] ${g.status} ${short(g.body, 900)}`); } // /models vs /models/user { const all: any[] = JSON.parse(readFileSync(`${OUT}models.json`, "utf8")).data; const user = await raw("/models/user"); const userIds = new Set(((user.body as any)?.data ?? []).map((m: any) => m.id)); const missing = all.filter((m) => !userIds.has(m.id)).map((m) => m.id); results.models_user_diff = { allCount: all.length, userCount: userIds.size, missingFromUser: missing }; console.log(`\n[/models/user] ${user.status} user=${userIds.size} all=${all.length} missing=${JSON.stringify(missing)}`); } save("09-reasoning-roundtrip.json", results);