spb/polyllm

1// Probe 02: generationConfig parameter acceptance matrix per model. One request per (model, param).2import { ai, MODELS, SUFFIX, save, errInfo, shortText, withRetry, sleep, PACE_MS } from "./lib.js";34const schema = { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] };56const CASES: Record<string, Record<string, unknown>> = {7  temperature_0: { temperature: 0 },8  temperature_1_5: { temperature: 1.5 },9  temperature_2: { temperature: 2 },10  temperature_2_5_out_of_range: { temperature: 2.5 },11  topP_0_9: { topP: 0.9 },12  topK_40: { topK: 40 },13  seed_42: { seed: 42 },14  stopSequences: { stopSequences: ["DONE"] },15  frequencyPenalty_0_5: { frequencyPenalty: 0.5 },16  presencePenalty_0_5: { presencePenalty: 0.5 },17  candidateCount_2: { candidateCount: 2 },18  responseMimeType_json: { responseMimeType: "application/json" },19  responseSchema: { responseMimeType: "application/json", responseSchema: { type: "OBJECT", properties: { answer: { type: "STRING" } }, required: ["answer"] } },20  responseJsonSchema: { responseMimeType: "application/json", responseJsonSchema: schema },21  responseMimeType_enum: { responseMimeType: "text/x.enum", responseSchema: { type: "STRING", enum: ["red", "blue", "green"] } },22  thinkingBudget_0: { thinkingConfig: { thinkingBudget: 0 } },23  thinkingBudget_1024: { thinkingConfig: { thinkingBudget: 1024 } },24  thinkingBudget_minus1: { thinkingConfig: { thinkingBudget: -1 } },25  includeThoughts: { thinkingConfig: { includeThoughts: true } },26  thinkingLevel_minimal: { thinkingConfig: { thinkingLevel: "MINIMAL" } },27  thinkingLevel_low: { thinkingConfig: { thinkingLevel: "LOW" } },28  thinkingLevel_medium: { thinkingConfig: { thinkingLevel: "MEDIUM" } },29  thinkingLevel_high: { thinkingConfig: { thinkingLevel: "HIGH" } },30  thinkingLevel_and_budget: { thinkingConfig: { thinkingLevel: "LOW", thinkingBudget: 512 } },31  responseLogprobs: { responseLogprobs: true, logprobs: 2 },32};3334const only = MODELS;35const onlyCases = process.argv[3] ? new Set(process.argv[3].split(",")) : null;36const results: any = {};3738for (const model of only) {39  results[model] = {};40  for (const [name, cfg] of Object.entries(CASES)) {41    if (onlyCases && !onlyCases.has(name)) continue;42    const t0 = Date.now();43    try {44      const res = await withRetry(() => ai.models.generateContent({45        model,46        contents: name.startsWith("responseMimeType_enum") ? "Which colour is the sky? Answer with one of the enum values." : "Answer in <= 5 words: what is 2+2? Then write DONE.",47        config: { maxOutputTokens: 1500, ...cfg } as any,48      }));49      const usage = res.usageMetadata ?? {};50      results[model][name] = {51        ok: true,52        text: shortText(res),53        finishReason: res.candidates?.[0]?.finishReason,54        thoughtsTokenCount: usage.thoughtsTokenCount ?? 0,55        candidatesTokenCount: usage.candidatesTokenCount,56        nCandidates: res.candidates?.length,57        hasThoughtParts: res.candidates?.[0]?.content?.parts?.some((p: any) => p.thought) ?? false,58        ms: Date.now() - t0,59      };60    } catch (e) {61      results[model][name] = { ok: false, error: errInfo(e), ms: Date.now() - t0 };62    }63    await sleep(PACE_MS);64    const r = results[model][name];65    console.log(model.padEnd(24), name.padEnd(30), r.ok ? `OK  thoughts=${r.thoughtsTokenCount} fin=${r.finishReason} "${r.text.slice(0, 40).replace(/\n/g, " ")}"` : `FAIL ${r.error.httpStatus} ${r.error.status}: ${r.error.message.slice(0, 160)}`);66  }67}68save(`02-params${SUFFIX}${onlyCases ? "-subset-" + Date.now() : ""}.json`, results);69