Repository Public Research preview
1// Live end-to-end tests through the real MCP protocol (stdio, official client).2// Run: ANTHROPIC_API_KEY=… node --test test/live.test.mjs (spends a few cents; skipped without a key)3import assert from "node:assert/strict";4import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";5import { tmpdir } from "node:os";6import { dirname, join } from "node:path";7import { after, before, test } from "node:test";8import { fileURLToPath } from "node:url";9import { deflateSync, crc32 } from "node:zlib";10import { Client } from "@modelcontextprotocol/sdk/client/index.js";11import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";1213const KEY = process.env.ANTHROPIC_API_KEY;14const skip = !KEY && "ANTHROPIC_API_KEY not set";15// Tests that need no key still run without one (protocol, offline tools, symlinked start).16const root = dirname(dirname(fileURLToPath(import.meta.url)));17const M = "claude-haiku-5-5";18const work = mkdtempSync(join(tmpdir(), "claude-api-mcp-"));19let client, offline;20const seen = [];2122async function connect(env) {23 const c = new Client({ name: "live-test", version: "1.0.0" });24 await c.connect(new StdioClientTransport({ command: process.execPath, args: [join(root, "server.mjs")], env: { PATH: process.env.PATH, CLAUDE_API_MCP_FILE_ROOT: work, ...env }, stderr: "pipe" }));25 return c;26}27async function call(name, args = {}, c = client) {28 const r = await c.callTool({ name, arguments: args }, undefined, { timeout: 600_000 });29 seen.push(JSON.stringify(r));30 return { data: r.structuredContent, isError: !!r.isError };31}32function png(r, g, b, size = 32) {33 const chunk = (type, data) => { const len = Buffer.alloc(4); len.writeUInt32BE(data.length); const td = Buffer.concat([Buffer.from(type), data]); const crc = Buffer.alloc(4); crc.writeUInt32BE(crc32(td) >>> 0); return Buffer.concat([len, td, crc]); };34 const ihdr = Buffer.alloc(13); ihdr.writeUInt32BE(size, 0); ihdr.writeUInt32BE(size, 4); ihdr[8] = 8; ihdr[9] = 2;35 const row = Buffer.concat([Buffer.from([0]), Buffer.alloc(size * 3).map((_, i) => [r, g, b][i % 3])]);36 return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk("IHDR", ihdr), chunk("IDAT", deflateSync(Buffer.concat(Array(size).fill(row)))), chunk("IEND", Buffer.alloc(0))]);37}38function pdf(text) {39 const objs = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", null, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"];40 const stream = `BT /F1 18 Tf 72 720 Td (${text}) Tj ET`;41 objs[3] = `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`;42 let out = "%PDF-1.4\n"; const offs = [];43 objs.forEach((o, i) => { offs.push(out.length); out += `${i + 1} 0 obj\n${o}\nendobj\n`; });44 const xref = out.length;45 out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n${offs.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join("")}trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;46 return Buffer.from(out, "latin1");47}48const user = (content) => [{ role: "user", content }];4950before(async () => { if (!skip) client = await connect({ ANTHROPIC_API_KEY: KEY }); offline = await connect({}); });51after(async () => { await client?.close(); await offline?.close(); rmSync(work, { recursive: true, force: true }); });5253test("protocol: tools are listed with schemas and safety annotations", async () => {54 const { tools } = await offline.listTools();55 assert.equal(tools.length, 37);56 for (const t of tools) { assert.ok(t.description.length > 40, t.name); assert.equal(t.inputSchema.type, "object"); assert.ok(t.annotations, t.name); }57 const by = Object.fromEntries(tools.map((t) => [t.name, t]));58 for (const n of ["file_delete", "batch_cancel", "agent_archive", "session_archive", "environment_archive"]) assert.equal(by[n].annotations.destructiveHint, true, n);59 for (const n of ["lint_request", "estimate_cost", "model_reference", "scaffold_agent"]) assert.equal(by[n].annotations.readOnlyHint, true, n);60 assert.match(by.create_message.description, /BILLED/);61 assert.match(offline.getInstructions(), /lint_request/);62});6364test("the server starts when launched through a symlinked path", async () => {65 const link = join(work, "linked");66 symlinkSync(root, link);67 const c = new Client({ name: "symlink-test", version: "1.0.0" });68 await c.connect(new StdioClientTransport({ command: process.execPath, args: [join(link, "server.mjs")], env: { PATH: process.env.PATH }, stderr: "pipe" }));69 assert.equal((await c.listTools()).tools.length, 37);70 await c.close();71});7273test("offline tools work without a key; live tools fail clearly without one", async () => {74 const ref = await call("model_reference", {}, offline);75 assert.equal(ref.data.models.length, 15);76 const one = await call("model_reference", { model: "claude-opus-5-5", detail: "full" }, offline);77 assert.equal(one.data.forced_tool_choice, false);78 const lint = await call("lint_request", { request: { model: "claude-opus-5-5", max_tokens: 100, temperature: 1, messages: user("x") } }, offline);79 assert.equal(lint.data.valid, false);80 assert.match(lint.data.findings[0].api_error, /Extra inputs/);81 const mig = await call("migrate_request", { request: { model: "claude-sonnet-4-5", max_tokens: 4000, temperature: 0.3, thinking: { type: "enabled", budget_tokens: 2000 }, tool_choice: { type: "any" }, tools: [{ name: "t", description: "A tool used by the live test.", input_schema: { type: "object", properties: {} } }], messages: user("x") }, target_model: "claude-sonnet-5-5" }, offline);82 assert.equal(mig.data.remaining.valid, true);83 assert.equal(mig.data.request.model, "claude-sonnet-5-5");84 assert.ok(mig.data.changes.length >= 3);85 assert.equal((await call("estimate_cost", { model: "claude-opus-5-5", input_tokens: 1000000, output_tokens: 1000000 }, offline)).data.usd_total, 24);86 assert.equal((await call("compare_costs", { input_tokens: 1000, output_tokens: 1000 }, offline)).data.ranking[0].model, "claude-haiku-5-5");87 assert.equal((await call("recommend_model", { task: "classify support tickets into 5 labels", volume: "high" }, offline)).data.model, "claude-haiku-5-5");88 assert.equal((await call("recommend_model", { task: "autonomous multi-step refactor of a large codebase", priority: "quality" }, offline)).data.model, "claude-fable-5-2");89 const req = await call("recommend_model", { task: "generate JSON", requires: ["forced_tool_choice"], priority: "quality" }, offline);90 assert.equal(req.data.model, "claude-opus-5");91 assert.match((await call("scaffold_agent", { approach: "manual_loop", language: "python", model: "claude-opus-5-5" }, offline)).data.code, /tool_result/);92 const ex = await call("explain_error", { status: 400, message: 'tool_choice: type "tool" and "any" are not supported for this model.' }, offline);93 assert.equal(ex.data.matches[0].cause, "tool_any");94 assert.match((await call("feature_guide", { topic: "thinking" }, offline)).data.guide, /between_tools/);95 const noKey = await call("list_models", {}, offline);96 assert.equal(noKey.isError, true);97 assert.match(noKey.data.error.message, /ANTHROPIC_API_KEY/);98 const bad = await offline.callTool({ name: "estimate_cost", arguments: { model: "claude-opus-5-5", input_tokens: -5 } }).catch((e) => ({ isError: true, thrown: String(e) }));99 assert.equal(bad.isError, true); // schema validation refuses negative tokens100});101102test("live: models and token counting", { skip }, async () => {103 const list = await call("list_models");104 assert.ok(list.data.count >= 10);105 const opus = list.data.models.find((m) => m.id === "claude-opus-5-5");106 assert.equal(opus.verified_by_probes, true);107 assert.equal(opus.context_window, 1000000);108 assert.equal((await call("get_model", { model: M })).data.max_tokens, 128000);109 const ct = await call("count_tokens", { request: { model: M, system: "You are terse.", messages: user("How many tokens is this sentence?"), max_tokens: 5 } });110 assert.ok(ct.data.input_tokens > 5 && ct.data.input_tokens < 60);111 assert.ok(ct.data.input_cost_usd > 0);112 const unknown = await call("get_model", { model: "claude-does-not-exist" });113 assert.equal(unknown.isError, true);114 assert.equal(unknown.data.error.status, 404);115});116117test("live: create_message lints, fixes, sends, and reports usage and cost", { skip }, async () => {118 const blocked = await call("create_message", { request: { model: "claude-opus-5-5", max_tokens: 50, thinking: { type: "disabled" }, messages: user("hi") } });119 assert.equal(blocked.data.sent, false);120 assert.ok(blocked.data.lint.findings.some((f) => f.code === "thinking-disable-unsupported"));121 const fixed = await call("create_message", { auto_fix: true, request: { model: M, max_tokens: 200, temperature: 0, effort: "low", messages: user("Reply with exactly: pong") } });122 assert.equal(fixed.data.sent, true);123 assert.ok(fixed.data.applied_fixes.length >= 2);124 assert.match(fixed.data.text.toLowerCase(), /pong/);125 assert.equal(fixed.data.stop_reason, "end_turn");126 assert.ok(fixed.data.usage.output_tokens > 0 && fixed.data.estimated_cost_usd > 0);127 const guard = await call("create_message", { request: { model: M, max_tokens: 100000, messages: user("hi") } });128 assert.equal(guard.isError, true);129 assert.match(guard.data.error.message, /spend guard/);130 // The linter said the API would reject it; skip_lint proves the API agrees.131 const forced = await call("create_message", { skip_lint: true, request: { model: "claude-opus-5-5", max_tokens: 50, thinking: { type: "disabled" }, messages: user("hi") } });132 assert.equal(forced.isError, true);133 assert.equal(forced.data.error.status, 400);134 assert.ok(forced.data.error.request_id);135});136137test("live: tools, strict schemas, thinking summary and mid-conversation system", { skip }, async () => {138 const tools = [{ name: "get_weather", description: "Get the current weather for a city. Always use it for weather questions.", strict: true, input_schema: { type: "object", properties: { city: { type: "string" } }, required: ["city"], additionalProperties: false } }];139 const first = await call("create_message", { request: { model: M, max_tokens: 1000, tools, messages: user("What is the weather in Paris? Use the tool.") } });140 assert.equal(first.data.stop_reason, "tool_use");141 const tc = first.data.tool_calls[0];142 assert.equal(tc.name, "get_weather");143 assert.match(tc.input.city, /Paris/);144 const second = await call("create_message", { request: { model: M, max_tokens: 1000, tools, messages: [...user("What is the weather in Paris? Use the tool."), { role: "assistant", content: first.data.content }, { role: "user", content: [{ type: "tool_result", tool_use_id: tc.id, content: "Rainy, 9°C" }] }] } });145 assert.equal(second.data.stop_reason, "end_turn");146 assert.match(second.data.text, /9/);147 const think = await call("create_message", { request: { model: "claude-sonnet-5-5", max_tokens: 3000, thinking: { type: "adaptive", display: "summarized" }, output_config: { effort: "medium" }, messages: user("A bat and a ball cost 1.10 in total. The bat costs 1.00 more than the ball. What does the ball cost? Answer with the number only.") } });148 assert.match(think.data.text, /0?\.05|5 cents/);149 const mid = await call("create_message", { request: { model: M, max_tokens: 200, messages: [...user("Say the word hello."), { role: "system", content: "Answer in uppercase letters only." }] } });150 assert.match(mid.data.text, /HELLO/);151});152153test("live: vision, PDF, citations and prompt caching", { skip }, async () => {154 const img = await call("create_message", { request: { model: M, max_tokens: 200, messages: user([{ type: "image", source: { type: "base64", media_type: "image/png", data: png(255, 0, 0).toString("base64") } }, { type: "text", text: "What single colour fills this image? One word." }]) } });155 assert.match(img.data.text.toLowerCase(), /red/);156 const doc = await call("create_message", { request: { model: M, max_tokens: 300, messages: user([{ type: "document", source: { type: "base64", media_type: "application/pdf", data: pdf("The access code is PINEAPPLE-42.").toString("base64") } }, { type: "text", text: "What is the access code in this PDF? Reply with the code only." }]) } });157 assert.match(doc.data.text, /PINEAPPLE-42/);158 const cite = await call("create_message", { request: { model: M, max_tokens: 300, messages: user([{ type: "document", source: { type: "text", media_type: "text/plain", data: "The registry was launched on 9 November 2026. It runs on PostgreSQL." }, title: "Notes", citations: { enabled: true } }, { type: "text", text: "When was the registry launched? One sentence." }]) } });159 const cited = cite.data.content.filter((b) => b.citations?.length);160 assert.ok(cited.length > 0);161 assert.match(cited[0].citations[0].cited_text, /9 November 2026/);162 const big = "You are a helpful assistant for the registry. " + Array.from({ length: 900 }, (_, i) => `Rule ${i}: always be precise about item number ${i}.`).join(" ");163 const req = { model: M, max_tokens: 50, system: [{ type: "text", text: big, cache_control: { type: "ephemeral" } }], messages: user("Reply: ok") };164 const a = await call("create_message", { request: req });165 const b = await call("create_message", { request: req });166 assert.ok((a.data.usage.cache_creation_input_tokens ?? 0) + (a.data.usage.cache_read_input_tokens ?? 0) > 4000, "prefix was not cached");167 assert.ok(b.data.usage.cache_read_input_tokens > 4000, "second call did not read the cache");168 assert.ok(b.data.estimated_cost_usd < a.data.estimated_cost_usd || a.data.usage.cache_read_input_tokens > 0);169});170171test("live: structured extraction and server-side web search", { skip }, async () => {172 const ex = await call("extract_structured", { model: M, prompt: "Invoice 4471 from Acme Corp, dated 2026-03-02, total 1,284.50 CAD, 3 line items.", schema: { type: "object", properties: { invoice_number: { type: "string" }, vendor: { type: "string" }, total: { type: "number" }, currency: { type: "string" }, line_items: { type: "integer" } } } });173 assert.equal(ex.data.ok, true);174 assert.deepEqual({ n: ex.data.data.invoice_number, v: ex.data.data.vendor, t: ex.data.data.total, c: ex.data.data.currency, l: ex.data.data.line_items }, { n: "4471", v: "Acme Corp", t: 1284.5, c: "CAD", l: 3 });175 const web = await call("create_message", { request: { model: M, max_tokens: 1500, tools: [{ type: "web_search_20260209", name: "web_search", max_uses: 1 }], messages: user("Search the web for the Model Context Protocol specification and give me the URL of its official site. One line.") } });176 assert.ok(web.data.content.some((b) => b.type === "server_tool_use"), "no server tool use");177 assert.match(web.data.text, /modelcontextprotocol/i);178 assert.ok(web.data.content.some((b) => b.type === "web_search_tool_result"), "no search result block");179});180181test("live: beta features go through request.betas", { skip }, async () => {182 const missing = await call("create_message", { request: { model: M, max_tokens: 200, output_config: { task_budget: { type: "tokens", total: 20000 } }, messages: user("Reply: ok") } });183 assert.equal(missing.data.sent, false);184 const tb = await call("create_message", { auto_fix: true, request: { model: M, max_tokens: 200, output_config: { task_budget: { type: "tokens", total: 20000 } }, messages: user("Reply: ok") } });185 assert.equal(tb.data.sent, true);186 assert.ok(tb.data.applied_fixes.some((c) => /task-budgets/.test(c)));187 const fb = await call("create_message", { request: { model: "claude-sonnet-5-5", max_tokens: 200, betas: ["server-side-fallback-2026-07-01"], fallbacks: "default", output_config: { effort: "low" }, messages: user("Reply: ok") } });188 assert.equal(fb.data.stop_reason, "end_turn");189});190191test("live: files — upload, reference, list, delete with confirmation; path confinement", { skip }, async () => {192 writeFileSync(join(work, "notes.txt"), "The launch codename is OSPREY.");193 const up = await call("file_upload", { path: "notes.txt" });194 assert.match(up.data.id, /^file_/);195 assert.equal(up.data.reference_block.type, "document");196 const outside = await call("file_upload", { path: "../../../../etc/hosts" });197 assert.equal(outside.isError, true);198 assert.match(outside.data.error.message, /outside the allowed root|ENOENT/);199 const both = await call("file_upload", { path: "notes.txt", content: "x" });200 assert.equal(both.isError, true);201 const ask = await call("create_message", { request: { model: M, max_tokens: 200, messages: user([up.data.reference_block, { type: "text", text: "What is the launch codename? One word." }]) } });202 assert.match(ask.data.text, /OSPREY/);203 const inline = await call("file_upload", { content: "inline body", filename: "inline.txt" });204 assert.ok((await call("file_list", { limit: 50 })).data.files.some((f) => f.id === up.data.id));205 assert.equal((await call("file_get", { id: up.data.id })).data.filename, "notes.txt");206 const noConfirm = await call("file_delete", { id: up.data.id });207 assert.equal(noConfirm.isError, true);208 assert.match(noConfirm.data.error.message, /confirm: true/);209 for (const id of [up.data.id, inline.data.id]) assert.equal((await call("file_delete", { id, confirm: true })).data.deleted, true);210 assert.equal((await call("file_get", { id: up.data.id })).isError, true);211});212213test("live: batches — lint gate, create, poll, results, cancel", { skip }, async () => {214 const bad = await call("batch_create", { requests: [{ custom_id: "a", params: { model: M, max_tokens: 50, speed: "fast", messages: user("x") } }] });215 assert.equal(bad.data.submitted, false);216 const dup = await call("batch_create", { requests: [{ custom_id: "a", params: { model: M, max_tokens: 50, messages: user("x") } }, { custom_id: "a", params: { model: M, max_tokens: 50, messages: user("y") } }] });217 assert.equal(dup.isError, true);218 const b = await call("batch_create", { requests: ["two", "three"].map((w) => ({ custom_id: `q-${w}`, params: { model: M, max_tokens: 100, messages: user(`Reply with exactly the digit for the number ${w}.`) } })) });219 assert.equal(b.data.submitted, true);220 const victim = await call("batch_create", { requests: [{ custom_id: "c", params: { model: M, max_tokens: 50, messages: user("x") } }] });221 assert.equal((await call("batch_cancel", { id: victim.data.id })).isError, true);222 assert.ok(["canceling", "ended"].includes((await call("batch_cancel", { id: victim.data.id, confirm: true })).data.processing_status));223 assert.ok((await call("batch_list", { limit: 5 })).data.batches.some((x) => x.id === b.data.id));224 assert.equal((await call("batch_results", { id: b.data.id })).data.ready === false || true, true);225 let status;226 for (let i = 0; i < 60; i++) { status = (await call("batch_get", { id: b.data.id })).data; if (status.processing_status === "ended") break; await new Promise((r) => setTimeout(r, 10_000)); }227 assert.equal(status.processing_status, "ended", "batch did not finish within 10 minutes");228 const res = await call("batch_results", { id: b.data.id });229 assert.equal(res.data.count, 2);230 const byId = Object.fromEntries(res.data.results.map((r) => [r.custom_id, r]));231 assert.match(byId["q-two"].text, /2/);232 assert.match(byId["q-three"].text, /3/);233 assert.ok(res.data.estimated_cost_usd > 0);234});235236test("live: Managed Agents — environment, agent, session run, events, archive", { skip }, async () => {237 const env = await call("environment_create", { name: `mcp-test-env-${Date.now()}` });238 assert.match(env.data.id, /^env/);239 const agent = await call("agent_create", { name: `mcp-test-agent-${Date.now()}`, model: M, system: "You work in a sandbox. Be brief." });240 assert.match(agent.data.id, /^agent/);241 assert.ok((await call("agent_list", { limit: 50 })).data.agents.some((a) => a.id === agent.data.id));242 assert.equal((await call("agent_get", { id: agent.data.id })).data.name, agent.data.name);243 assert.ok((await call("environment_list", { limit: 50 })).data.environments.some((e) => e.id === env.data.id));244 const missing = await call("session_run", { message: "hi" });245 assert.equal(missing.isError, true);246 const run = await call("session_run", { agent_id: agent.data.id, environment_id: env.data.id, message: "Use bash to compute 17*23 and tell me the result.", timeout_seconds: 300 });247 assert.equal(run.isError, false, JSON.stringify(run.data));248 assert.ok(["idle", "requires_action"].includes(run.data.status), run.data.status);249 if (run.data.status === "idle") assert.match(run.data.text, /391/);250 const sid = run.data.session_id;251 assert.ok((await call("session_events", { session_id: sid })).data.events.some((e) => e.type === "user.message"));252 assert.equal((await call("session_get", { id: sid })).data.id, sid);253 assert.ok((await call("session_list", { limit: 20 })).data.sessions.some((s) => s.id === sid));254 assert.equal((await call("session_archive", { id: sid })).isError, true);255 await call("session_archive", { id: sid, confirm: true });256 assert.equal((await call("agent_archive", { id: agent.data.id })).isError, true);257 assert.ok((await call("agent_archive", { id: agent.data.id, confirm: true })).data.archived_at);258 assert.ok((await call("environment_archive", { id: env.data.id, confirm: true })).data.archived_at);259});260261test("the API key never appears in any tool output", { skip }, () => {262 const all = seen.join("\n");263 assert.ok(seen.length > 60);264 assert.equal(all.includes(KEY), false);265 assert.equal(/sk-ant-[A-Za-z0-9_-]{20,}/.test(all), false);266});267