Repository Public Research preview
1// Request builders shared by the live probes and by the linter consistency test.2// Each entry: [api, build(model)] where api is "responses" or "chat".3import { deflateSync, crc32 } from "node:zlib";45function png(r, g, b, size = 32) {6 const chunk = (type, data) => { const len = Buffer.alloc(4); len.writeUInt32BE(data.length); const td = Buffer.concat([Buffer.from(type), data]); const crc = Buffer.alloc(4); crc.writeUInt32BE(crc32(td) >>> 0); return Buffer.concat([len, td, crc]); };7 const ihdr = Buffer.alloc(13); ihdr.writeUInt32BE(size, 0); ihdr.writeUInt32BE(size, 4); ihdr[8] = 8; ihdr[9] = 2;8 const row = Buffer.concat([Buffer.from([0]), Buffer.alloc(size * 3).map((_, i) => [r, g, b][i % 3])]);9 return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk("IHDR", ihdr), chunk("IDAT", deflateSync(Buffer.concat(Array(size).fill(row)))), chunk("IEND", Buffer.alloc(0))]);10}11export const RED_PNG = `data:image/png;base64,${png(255, 0, 0).toString("base64")}`;12export const pngDataUrl = (r, g, b, size = 256) => `data:image/png;base64,${png(r, g, b, size).toString("base64")}`;13const Q = "Reply with the single word: ok";14export const fn = { type: "function", name: "get_time", description: "Return the current time in a timezone.", parameters: { type: "object", properties: { tz: { type: "string" } }, required: ["tz"], additionalProperties: false }, strict: true };15const chatFn = { type: "function", function: { name: "get_time", description: "Return the current time in a timezone.", parameters: fn.parameters, strict: true } };16const schema = { type: "object", properties: { answer: { type: "string" } }, required: ["answer"], additionalProperties: false };17const R = (extra) => (m) => ({ model: m, input: Q, max_output_tokens: 64, ...extra });18const C = (extra) => (m) => ({ model: m, messages: [{ role: "user", content: Q }], max_completion_tokens: 64, ...extra });1920export const PROBES = {21 basic: ["responses", R({})],22 instructions: ["responses", R({ instructions: "Be terse." })],23 effort_none: ["responses", R({ reasoning: { effort: "none" } })],24 effort_minimal: ["responses", R({ reasoning: { effort: "minimal" } })],25 effort_low: ["responses", R({ reasoning: { effort: "low" } })],26 effort_medium: ["responses", R({ reasoning: { effort: "medium" } })],27 effort_high: ["responses", R({ reasoning: { effort: "high" } })],28 effort_xhigh: ["responses", R({ reasoning: { effort: "xhigh" } })],29 effort_max: ["responses", R({ reasoning: { effort: "max" } })],30 reasoning_summary: ["responses", R({ reasoning: { summary: "auto" } })],31 temperature: ["responses", R({ temperature: 0.5 })],32 top_p: ["responses", R({ top_p: 0.9 })],33 verbosity_low: ["responses", R({ text: { verbosity: "low" } })],34 verbosity_high: ["responses", R({ text: { verbosity: "high" } })],35 structured: ["responses", (m) => ({ model: m, input: "Answer in JSON: what is 2+2?", max_output_tokens: 128, text: { format: { type: "json_schema", name: "answer", schema, strict: true } } })],36 json_object: ["responses", (m) => ({ model: m, input: "Answer in JSON: what is 2+2?", max_output_tokens: 128, text: { format: { type: "json_object" } } })],37 tool_auto: ["responses", (m) => ({ model: m, input: "What time is it in UTC? Use the tool.", max_output_tokens: 128, tools: [fn], tool_choice: "auto" })],38 tool_required: ["responses", (m) => ({ model: m, input: "What time is it in UTC?", max_output_tokens: 128, tools: [fn], tool_choice: "required" })],39 tool_forced: ["responses", (m) => ({ model: m, input: "What time is it in UTC?", max_output_tokens: 128, tools: [fn], tool_choice: { type: "function", name: "get_time" } })],40 tool_none: ["responses", R({ tools: [fn], tool_choice: "none" })],41 parallel_false: ["responses", R({ tools: [fn], parallel_tool_calls: false })],42 web_search: ["responses", R({ tools: [{ type: "web_search" }], tool_choice: "none" })],43 web_search_preview: ["responses", R({ tools: [{ type: "web_search_preview" }], tool_choice: "none" })],44 code_interpreter: ["responses", R({ tools: [{ type: "code_interpreter", container: { type: "auto" } }], tool_choice: "none" })],45 role_system: ["responses", (m) => ({ model: m, max_output_tokens: 64, input: [{ role: "system", content: "Be terse." }, { role: "user", content: Q }] })],46 role_developer: ["responses", (m) => ({ model: m, max_output_tokens: 64, input: [{ role: "developer", content: "Be terse." }, { role: "user", content: Q }] })],47 image_input: ["responses", (m) => ({ model: m, max_output_tokens: 64, input: [{ role: "user", content: [{ type: "input_image", image_url: RED_PNG }, { type: "input_text", text: "One word: what colour is this square?" }] }] })],48 store_false: ["responses", R({ store: false })],49 truncation_auto: ["responses", R({ truncation: "auto" })],50 service_tier_flex: ["responses", R({ service_tier: "flex" })],51 service_tier_priority: ["responses", R({ service_tier: "priority" })],52 prompt_cache_key: ["responses", R({ prompt_cache_key: "probe-key-1" })],53 cache_retention_24h: ["responses", R({ prompt_cache_retention: "24h" })],54 max_output_8: ["responses", (m) => ({ model: m, input: Q, max_output_tokens: 8 })],55 include_encrypted: ["responses", R({ store: false, include: ["reasoning.encrypted_content"] })],56 top_logprobs: ["responses", R({ top_logprobs: 2, include: ["message.output_text.logprobs"] })],57 metadata: ["responses", R({ metadata: { purpose: "probe" } })],58 max_tool_calls: ["responses", R({ tools: [fn], max_tool_calls: 1 })],59 background: ["responses", R({ background: true })],60 temperature_effort_low: ["responses", R({ temperature: 0.5, reasoning: { effort: "low" } })],61 temperature_effort_none: ["responses", R({ temperature: 0.5, reasoning: { effort: "none" } })],62 chat_tools_effort_none: ["chat", (m) => ({ model: m, messages: [{ role: "user", content: "What time is it in UTC? Use the tool." }], max_completion_tokens: 128, tools: [chatFn], reasoning_effort: "none" })],63 chat_temperature_effort_none: ["chat", C({ temperature: 0.5, reasoning_effort: "none" })],64 chat_basic: ["chat", C({})],65 chat_max_tokens_legacy: ["chat", (m) => ({ model: m, messages: [{ role: "user", content: Q }], max_tokens: 64 })],66 chat_temperature: ["chat", C({ temperature: 0.5 })],67 chat_reasoning_effort: ["chat", C({ reasoning_effort: "low" })],68 chat_json_schema: ["chat", (m) => ({ model: m, messages: [{ role: "user", content: "Answer in JSON: what is 2+2?" }], max_completion_tokens: 128, response_format: { type: "json_schema", json_schema: { name: "answer", schema, strict: true } } })],69 chat_tools: ["chat", (m) => ({ model: m, messages: [{ role: "user", content: "What time is it in UTC? Use the tool." }], max_completion_tokens: 128, tools: [chatFn] })],70 chat_n2: ["chat", C({ n: 2 })],71 chat_stop: ["chat", C({ stop: ["STOP"] })],72 chat_seed: ["chat", C({ seed: 7 })],73 chat_logprobs: ["chat", C({ logprobs: true, top_logprobs: 2 })],74 chat_system_role: ["chat", (m) => ({ model: m, max_completion_tokens: 64, messages: [{ role: "system", content: "Be terse." }, { role: "user", content: Q }] })],75 chat_developer_role: ["chat", (m) => ({ model: m, max_completion_tokens: 64, messages: [{ role: "developer", content: "Be terse." }, { role: "user", content: Q }] })],76};7778/** Text-generation models to probe (aliases; dated snapshots behave like their alias). */79export const TEXT_MODELS = [80 "gpt-6.1-sol", "gpt-6.1-terra", "gpt-6-sol", "gpt-6-luna", "gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna",81 "gpt-5.5", "gpt-5.5-pro", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", "gpt-5.4-pro", "gpt-5.3-codex", "gpt-5.2", "gpt-5.2-pro",82 "gpt-5.1", "gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-pro", "gpt-4.1", "gpt-4.1-mini", "gpt-4o", "gpt-4o-mini", "o3", "o3-pro", "chat-latest",83];84