spb/polyllm

1// Probe 03: per model — (d) function-calling round trip with streaming, (e) structured output via responseJsonSchema.2// Plus (h) googleSearch grounding and code execution on selected models.3import { ai, MODELS, SUFFIX, save, errInfo, shortText, withRetry, sleep, PACE_MS } from "./lib.js";45const results: any = {};6const weatherTool = {7  functionDeclarations: [{8    name: "get_weather",9    description: "Get the current weather for a city.",10    parametersJsonSchema: { type: "object", properties: { city: { type: "string" }, unit: { type: "string", enum: ["C", "F"] } }, required: ["city"] },11  }],12};1314for (const model of MODELS) {15  const r: any = (results[model] = {});1617  // (d) function calling, streaming, then send functionResponse18  try {19    const contents: any[] = [{ role: "user", parts: [{ text: "What is the weather in Montreal right now? Use the tool." }] }];20    const stream = await withRetry(() => ai.models.generateContentStream({ model, contents, config: { tools: [weatherTool], maxOutputTokens: 2000 } }));21    const chunks: any[] = [];22    let modelParts: any[] = [];23    for await (const c of stream) {24      const parts = c.candidates?.[0]?.content?.parts ?? [];25      chunks.push({ parts: parts.map((p: any) => ({ keys: Object.keys(p), fc: p.functionCall, thought: p.thought, sig: p.thoughtSignature ? `${p.thoughtSignature.length} chars` : undefined })), finishReason: c.candidates?.[0]?.finishReason, usage: c.usageMetadata ? Object.keys(c.usageMetadata) : null });26      modelParts.push(...parts);27    }28    const fcParts = modelParts.filter((p) => p.functionCall);29    r.functionCall = { ok: true, chunkCount: chunks.length, chunks, functionCalls: fcParts.map((p) => p.functionCall), sigOnFcPart: fcParts.map((p) => !!p.thoughtSignature) };30    if (fcParts.length) {31      // Round trip: append model content (with thoughtSignature preserved) and functionResponse parts32      contents.push({ role: "model", parts: modelParts });33      contents.push({34        role: "user",35        parts: fcParts.map((p) => ({ functionResponse: { name: p.functionCall.name, id: p.functionCall.id, response: { temperatureC: 21, condition: "sunny" } } })),36      });37      await sleep(PACE_MS);38      const res2 = await withRetry(() => ai.models.generateContent({ model, contents, config: { tools: [weatherTool], maxOutputTokens: 2000 } }));39      r.functionCall.roundTrip = { text: shortText(res2), finishReason: res2.candidates?.[0]?.finishReason, usage: res2.usageMetadata };40    }41  } catch (e) {42    r.functionCall = { ok: false, error: errInfo(e) };43  }4445  // (e) structured output responseJsonSchema46  try {47    await sleep(PACE_MS);48    const res = await withRetry(() => ai.models.generateContent({49      model,50      contents: "Extract: 'Alice is 30 and lives in Paris.'",51      config: {52        maxOutputTokens: 2000,53        responseMimeType: "application/json",54        responseJsonSchema: { type: "object", properties: { name: { type: "string" }, age: { type: "integer" }, city: { type: "string" } }, required: ["name", "age", "city"], additionalProperties: false },55      },56    }));57    const txt = shortText(res);58    let parsed: any = null; try { parsed = JSON.parse(res.candidates?.[0]?.content?.parts?.filter((p: any) => !p.thought).map((p: any) => p.text).join("") ?? ""); } catch {}59    r.structured = { ok: true, text: txt, parsedOk: !!parsed, parsed, finishReason: res.candidates?.[0]?.finishReason };60  } catch (e) {61    r.structured = { ok: false, error: errInfo(e) };62  }63  await sleep(PACE_MS);64  console.log(model, "fc:", r.functionCall.ok ? `${r.functionCall.functionCalls.length} call(s) chunks=${r.functionCall.chunkCount} rt="${r.functionCall.roundTrip?.text?.slice(0, 50)}"` : r.functionCall.error.message.slice(0, 200), "| structured:", r.structured.ok ? r.structured.text.slice(0, 80) : r.structured.error.message.slice(0, 200));65}6667const EXTRAS = process.argv[3] === "extras";68// (h) Google Search grounding (3.x models; 2.5 is 404 for this key)69for (const model of EXTRAS ? [MODELS[0]] : []) {70  await sleep(PACE_MS);71  try {72    const res = await withRetry(() => ai.models.generateContent({ model, contents: "Who won the most recent FIFA World Cup final and what was the score? One sentence.", config: { tools: [{ googleSearch: {} }], maxOutputTokens: 3000 } }));73    const gm: any = res.candidates?.[0]?.groundingMetadata;74    results[`search:${model}`] = {75      ok: true, text: shortText(res),76      groundingMetadataKeys: gm ? Object.keys(gm) : null,77      webSearchQueries: gm?.webSearchQueries,78      groundingChunksSample: gm?.groundingChunks?.slice(0, 2),79      groundingSupportsSample: gm?.groundingSupports?.slice(0, 2),80      searchEntryPointKeys: gm?.searchEntryPoint ? Object.keys(gm.searchEntryPoint) : null,81      renderedContentLen: gm?.searchEntryPoint?.renderedContent?.length,82      usage: res.usageMetadata,83      partKeys: res.candidates?.[0]?.content?.parts?.map((p: any) => Object.keys(p)),84    };85  } catch (e) { results[`search:${model}`] = { ok: false, error: errInfo(e) }; }86  console.log("search", model, JSON.stringify(results[`search:${model}`]).slice(0, 300));87}8889// Code execution on one model90if (EXTRAS) try {91  await sleep(PACE_MS);92  const res = await withRetry(() => ai.models.generateContent({ model: MODELS[0], contents: "Compute the 30th Fibonacci number with Python and report it.", config: { tools: [{ codeExecution: {} }], maxOutputTokens: 3000 } }));93  results[`codeExecution:${MODELS[0]}`] = { ok: true, parts: res.candidates?.[0]?.content?.parts?.map((p: any) => ({ keys: Object.keys(p), executableCode: p.executableCode, codeExecutionResult: p.codeExecutionResult, text: p.text?.slice(0, 100) })), usage: res.usageMetadata };94} catch (e) { results[`codeExecution:${MODELS[0]}`] = { ok: false, error: errInfo(e) }; }95if (EXTRAS) console.log("codeExec", JSON.stringify(results[`codeExecution:${MODELS[0]}`]).slice(0, 400));9697// toolConfig mode ANY (+ streamFunctionCallArguments is Vertex-only: SDK throws before sending) on 3.898if (EXTRAS) try {99  await sleep(PACE_MS);100  const stream = await withRetry(() => ai.models.generateContentStream({ model: MODELS[0], contents: "Weather in Quebec City?", config: { tools: [weatherTool], toolConfig: { functionCallingConfig: { mode: "ANY" as any, allowedFunctionNames: ["get_weather"] } }, maxOutputTokens: 2000 } }));101  const chunks: any[] = [];102  for await (const c of stream) chunks.push(c.candidates?.[0]?.content?.parts);103  results[`fcModeAny:${MODELS[0]}`] = { ok: true, chunks };104} catch (e) { results[`fcModeAny:${MODELS[0]}`] = { ok: false, error: errInfo(e) }; }105if (EXTRAS) console.log("fcAny", JSON.stringify(results[`fcModeAny:${MODELS[0]}`]).slice(0, 500));106107save(`03-tools${SUFFIX}.json`, results);108