Repository Public Research preview
1// Does the skill change outcomes? An author model writes Messages requests for target models, with2// and without the skill in its system prompt; every request is then sent to the live API.3// Score = share of requests that pass API validation (HTTP 400 = invalid; a 429 "rate limit of 0"4// means the request is valid but the account lacks access to that feature, and is counted separately). Usage: ANTHROPIC_API_KEY=… node eval.mjs [author-model]5import Anthropic from "../mcp/node_modules/@anthropic-ai/sdk/index.mjs";6import { readFileSync, writeFileSync } from "node:fs";78const client = new Anthropic({ maxRetries: 3 });9const author = process.argv[2] ?? "claude-haiku-5-5";10const skillDir = new URL("../skill/claude-api-toolkit/", import.meta.url);11const SKILL = ["SKILL.md", "references/request-rules.md", "references/requests.md", "references/tools-and-agents.md", "references/caching-and-context.md"].map((f) => `<file path="${f}">\n${readFileSync(new URL(f, skillDir), "utf8")}\n</file>`).join("\n\n");12const BASE = "You write request bodies for the Anthropic Messages API. Reply with ONE JSON object only: the request body. No prose, no code fence.";1314const TASKS = [15 ["opus55-deterministic", "claude-opus-5-5", "Classify the sentiment of 'great product' as positive/negative. The output must be as deterministic and repeatable as possible."],16 ["opus55-deep-thinking", "claude-opus-5-5", "Solve a hard logic puzzle. Give the model a generous extended-thinking budget of about 10000 tokens."],17 ["sonnet55-force-tool", "claude-sonnet-5-5", "Define a get_stock_price(ticker) tool and make sure the model calls it for the question 'What is AAPL trading at?'. It must not answer in plain text."],18 ["fable52-no-thinking", "claude-fable-5-2", "Answer 'What is the capital of France?' as fast and cheaply as possible, with thinking turned off."],19 ["sonnet55-no-thinking", "claude-sonnet-5-5", "Translate 'good morning' to French. Turn thinking off to save tokens."],20 ["opus55-json", "claude-opus-5-5", "Extract {name, age} from 'Marie is 34 years old'. Guarantee that the reply is valid JSON starting with '{'."],21 ["haiku45-thinking", "claude-haiku-4-5-20251001", "Work out 17*23 step by step with the model's thinking feature enabled."],22 ["opus46-max-effort", "claude-opus-4-6", "Review a tricky concurrency bug. Use the highest reasoning effort this model offers."],23 ["sonnet45-effort", "claude-sonnet-4-5-20250929", "Summarise a paragraph about tides in one sentence, using low effort to keep it cheap."],24 ["haiku55-creative", "claude-haiku-5-5", "Write a four-line poem about winter. Make it highly creative and varied between runs."],25 ["opus55-long-output", "claude-opus-5-5", "Write a detailed 3000-word technical design document for a URL shortener. Allow the largest output the model supports."],26 ["opus5-fast", "claude-opus-5", "Answer 'ping' with 'pong' using the model's fast output mode."],27 ["sonnet5-system-mid", "claude-sonnet-5", "A two-turn chat: the user said 'hi', the assistant said 'hello', the user now asks 'what is 2+2?'. Add an operator instruction for this last turn only: answer in French."],28 ["opus47-plain", "claude-opus-4-7", "Explain what a mutex is in two sentences."],29 ["fable51-tool-json", "claude-fable-5-1", "Return the fields {city, country} for 'Lyon' by forcing a call to a record_place tool."],30 ["haiku55-plain", "claude-haiku-5-5", "Say hello in one short sentence."],31];3233function parse(text) {34 const t = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/, "");35 const i = t.indexOf("{"), j = t.lastIndexOf("}");36 return JSON.parse(t.slice(i, j + 1));37}38async function writeRequest(task, withSkill) {39 const [, target, goal] = task;40 const r = await client.messages.create({41 model: author, max_tokens: 4000, output_config: { effort: "medium" },42 system: withSkill ? `${BASE}\n\nYou have this skill. Follow it.\n\n${SKILL}` : BASE,43 messages: [{ role: "user", content: `Target model: ${target}\nGoal: ${goal}` }],44 });45 return parse(r.content.find((b) => b.type === "text").text);46}47async function trySend(req) {48 const body = structuredClone(req);49 delete body.stream;50 if (Number.isInteger(body.max_tokens) && body.max_tokens > 1500 && !(body.thinking?.budget_tokens >= 1024)) body.max_tokens_original = undefined;51 const { betas, max_tokens_original: _x, ...rest } = body;52 try {53 // Validation happens before generation; a tiny generation is enough to learn accept/reject.54 const api = Array.isArray(betas) && betas.length ? client.beta.messages : client.messages;55 const s = api.stream(Array.isArray(betas) && betas.length ? { ...rest, betas } : rest);56 for await (const ev of s) { if (ev.type === "message_start") { s.abort(); break; } }57 return { accepted: true };58 } catch (e) {59 if (e?.name === "APIUserAbortError" || /abort/i.test(e?.message ?? "")) return { accepted: true };60 if (e.status === 429) return { accepted: true, no_access: true, status: 429 };61 return { accepted: false, status: e.status ?? null, message: String(e.error?.error?.message ?? e.message).replace(/org: [0-9a-f-]{36}/g, "org: <redacted>").slice(0, 160) };62 }63}6465const rows = [];66await Promise.all(TASKS.map(async (task) => {67 const row = { task: task[0], target: task[1] };68 for (const cond of ["without", "with"]) {69 try { const req = await writeRequest(task, cond === "with"); row[cond] = { ...(await trySend(req)), wrong_model: req.model !== task[1] || undefined }; }70 catch (e) { row[cond] = { accepted: false, status: "unparseable", message: String(e.message).slice(0, 120) }; }71 }72 rows.push(row);73}));74rows.sort((a, b) => TASKS.findIndex((t) => t[0] === a.task) - TASKS.findIndex((t) => t[0] === b.task));75const score = (c) => rows.filter((r) => r[c].accepted && !r[c].wrong_model).length;76for (const r of rows) console.log(r.task.padEnd(24), "without:", r.without.accepted ? "ok " : `${r.without.status} `, "with:", r.with.accepted ? "ok " : `${r.with.status} `, r.without.accepted ? "" : `| ${r.without.message}`, r.with.accepted ? "" : `|| WITH: ${r.with.message}`);77console.log(`\nauthor ${author}: accepted without skill ${score("without")}/${rows.length}, with skill ${score("with")}/${rows.length}`);78writeFileSync(new URL(`./eval-${author}.json`, import.meta.url), JSON.stringify({ author, date: new Date().toISOString(), tasks: rows.length, accepted_without: score("without"), accepted_with: score("with"), rows }, null, 2));79