Repository Public Research preview
1// Does the skill change outcomes? An author model writes OpenAI request bodies for target models,2// with and without the skill in its instructions; every request is then sent to the live API.3// Score = share of requests that pass API validation (HTTP 400/404 = invalid; 429 = valid, no capacity).4// Usage: OPENAI_API_KEY=… node eval.mjs [author-model]5import OpenAI from "../mcp/node_modules/openai/index.mjs";6import { readFileSync, writeFileSync } from "node:fs";7import { pngDataUrl } from "../mcp/probes/requests.mjs";89const client = new OpenAI({ maxRetries: 3, timeout: 180_000 });10const author = process.argv[2] ?? "gpt-6-luna";11const skillDir = new URL("../skill/openai-api-toolkit/", import.meta.url);12const SKILL = ["SKILL.md", "references/request-rules.md", "references/requests.md", "references/tools-and-agents.md"].map((f) => `<file path="${f}">\n${readFileSync(new URL(f, skillDir), "utf8")}\n</file>`).join("\n\n");13const BASE = "You write request bodies for the OpenAI API. Reply with ONE JSON object only: the request body (for the Responses API, or for Chat Completions when the task says so). No prose, no code fence.";1415const TASKS = [16 ["sol61-deterministic", "gpt-6.1-sol", "Responses API. Classify the sentiment of 'great product' as positive/negative. The output must be as deterministic and repeatable as possible."],17 ["luna6-low-temperature", "gpt-6-luna", "Responses API. Translate 'good morning' to French with a low sampling temperature of 0.2 for consistency."],18 ["astra6-fast-cheap", "gpt-6-astra", "Responses API. Answer 'What is the capital of France?' with the least reasoning the model allows, to be fast and cheap."],19 ["gpt5-no-reasoning", "gpt-5", "Responses API. Reply 'pong' to 'ping' with reasoning turned off or as low as it goes."],20 ["gpt55pro-chat", "gpt-5.5-pro", "A simple chat request: system prompt 'Be brief', user asks 'What is a mutex?'."],21 ["sol61-chat-tools", "gpt-6.1-sol", "Define a get_stock_price(ticker) function tool and ask 'What is AAPL trading at?' so the model calls it."],22 ["luna6-short-answer", "gpt-6-luna", "Chat Completions. Answer 'yes or no: is water wet?' and cap the answer at 20 tokens, stopping at the first newline."],23 ["gpt41-effort", "gpt-4.1", "Responses API. Solve a logic puzzle about three boxes with high reasoning effort."],24 ["sol61-json", "gpt-6.1-sol", "Responses API. Extract {name, age, email?} from 'Marie is 34'. The reply must be JSON guaranteed to match the schema; email is optional."],25 ["terra61-vision", "gpt-6.1-terra", "Responses API. Ask what colour the image at https://img.invalid/square.png is (use exactly that URL)."],26 ["o3-search", "o3", "Responses API. Use hosted web search and force the model to use a tool to find today's top science headline."],27 ["gpt5mini-xhigh", "gpt-5-mini", "Responses API. Review a tricky concurrency bug with the highest reasoning effort available."],28 ["gpt54-creative", "gpt-5.4", "Responses API. Write a four-line poem about winter with high reasoning effort and temperature 1.3 for variety."],29 ["codex53-chat", "gpt-5.3-codex", "Write a Python function that reverses a string. Use whatever API this model supports."],30 ["luna6-tool-result", "gpt-6-luna", "Responses API. Continue this exchange: the user asked for the weather in Paris, the model called get_weather with call id call_abc and arguments {\"city\":\"Paris\"}, and the tool returned 'Rainy, 9C'. Include the tool definition."],31 ["sol61-plain", "gpt-6.1-sol", "Responses API. Say hello in one short sentence."],32];3334function parse(text) {35 const t = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/, "");36 return JSON.parse(t.slice(t.indexOf("{"), t.lastIndexOf("}") + 1));37}38async function writeRequest(task, withSkill) {39 const r = await client.responses.create({ model: author, instructions: withSkill ? `${BASE}\n\nYou have this skill. Follow it.\n\n${SKILL}` : BASE, input: `Target model: ${task[1]}\nTask: ${task[2]}`, max_output_tokens: 6000 });40 return parse(r.output_text);41}42async function trySend(req) {43 // The fixture URL is swapped for a real image so the test measures request validity, not network reachability.44 const body = { ...JSON.parse(JSON.stringify(req).replaceAll("https://img.invalid/square.png", pngDataUrl(0, 0, 255))), stream: true };45 delete body.stream_options;46 try {47 // Validation happens before generation: the first stream event is enough to learn accept/reject.48 const s = "messages" in body ? await client.chat.completions.create(body) : await client.responses.create(body);49 for await (const _ of s) { s.controller.abort(); break; }50 return { accepted: true };51 } catch (e) {52 if (e?.name === "APIUserAbortError" || /abort/i.test(e?.message ?? "")) return { accepted: true };53 if (e.status === 429) return { accepted: true, no_capacity: true };54 return { accepted: false, status: e.status ?? null, message: String(e.error?.message ?? e.message).replace(/org-[A-Za-z0-9]+/g, "org-<redacted>").slice(0, 170) };55 }56}5758const rows = [];59await Promise.all(TASKS.map(async (task) => {60 const row = { task: task[0], target: task[1] };61 for (const cond of ["without", "with"]) {62 try { const req = await writeRequest(task, cond === "with"); row[cond] = { ...(await trySend(req)), wrong_model: req.model !== task[1] || undefined, api: "messages" in req ? "chat" : "responses" }; }63 catch (e) { row[cond] = { accepted: false, status: "unparseable", message: String(e.message).slice(0, 120) }; }64 }65 rows.push(row);66}));67rows.sort((a, b) => TASKS.findIndex((t) => t[0] === a.task) - TASKS.findIndex((t) => t[0] === b.task));68const score = (c) => rows.filter((r) => r[c].accepted && !r[c].wrong_model).length;69for (const r of rows) console.log(r.task.padEnd(22), "without:", r.without.accepted ? "ok " : `${r.without.status} `, "with:", r.with.accepted ? "ok " : `${r.with.status} `, r.without.accepted ? "" : `| ${r.without.message}`, r.with.accepted ? "" : `|| WITH: ${r.with.message}`);70console.log(`\nauthor ${author}: valid without skill ${score("without")}/${rows.length}, with skill ${score("with")}/${rows.length}`);71writeFileSync(new URL(`./eval-${author}.json`, import.meta.url), JSON.stringify({ author, date: new Date().toISOString(), tasks: rows.length, valid_without: score("without"), valid_with: score("with"), rows }, null, 2));72