Repository Public Research preview
1// Live end-to-end tests through the real MCP protocol (stdio, official client).2// Run: OPENAI_API_KEY=… node --test test/live.test.mjs (spends a few cents; live tests skip without a key)3import assert from "node:assert/strict";4import { existsSync, mkdtempSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";5import { tmpdir } from "node:os";6import { dirname, join } from "node:path";7import { after, before, test } from "node:test";8import { fileURLToPath } from "node:url";9import { Client } from "@modelcontextprotocol/sdk/client/index.js";10import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";11import { pngDataUrl } from "../probes/requests.mjs";1213const KEY = process.env.OPENAI_API_KEY;14const skip = !KEY && "OPENAI_API_KEY not set";15const root = dirname(dirname(fileURLToPath(import.meta.url)));16const M = "gpt-6-luna";17const work = mkdtempSync(join(tmpdir(), "openai-api-mcp-"));18let client, offline;19const seen = [];2021async function connect(env, entry = join(root, "server.mjs")) {22 const c = new Client({ name: "live-test", version: "1.0.0" });23 await c.connect(new StdioClientTransport({ command: process.execPath, args: [entry], env: { PATH: process.env.PATH, OPENAI_API_MCP_FILE_ROOT: work, ...env }, stderr: "pipe" }));24 return c;25}26async function call(name, args = {}, c = client) {27 const r = await c.callTool({ name, arguments: args }, undefined, { timeout: 900_000 });28 seen.push(JSON.stringify(r));29 return { data: r.structuredContent, isError: !!r.isError };30}3132before(async () => { if (!skip) client = await connect({ OPENAI_API_KEY: KEY }); offline = await connect({}); });33after(async () => { await client?.close(); await offline?.close(); rmSync(work, { recursive: true, force: true }); });3435test("protocol: tools are listed with schemas and safety annotations", async () => {36 const { tools } = await offline.listTools();37 assert.equal(tools.length, 37);38 for (const t of tools) { assert.ok(t.description.length > 40, t.name); assert.equal(t.inputSchema.type, "object"); assert.ok(t.annotations, t.name); }39 const by = Object.fromEntries(tools.map((t) => [t.name, t]));40 for (const n of ["file_delete", "batch_cancel", "delete_response", "cancel_response", "vector_store_delete"]) assert.equal(by[n].annotations.destructiveHint, true, n);41 for (const n of ["lint_request", "estimate_cost", "model_reference", "scaffold_app", "chat_to_responses"]) assert.equal(by[n].annotations.readOnlyHint, true, n);42 assert.match(by.create_response.description, /BILLED/);43 assert.match(offline.getInstructions(), /lint_request/);44});4546test("the server starts when launched through a symlinked path", async () => {47 const link = join(work, "linked");48 symlinkSync(root, link);49 const c = await connect({}, join(link, "server.mjs"));50 assert.equal((await c.listTools()).tools.length, 37);51 await c.close();52});5354test("offline tools work without a key; live tools fail clearly without one", async () => {55 assert.equal((await call("model_reference", {}, offline)).data.models.length, 29);56 const one = await call("model_reference", { model: "gpt-6.1-sol", detail: "full" }, offline);57 assert.equal(one.data.context_window, 1050000);58 assert.equal(one.data.responses.sampling_params, "never");59 const lint = await call("lint_request", { request: { model: "gpt-6.1-sol", input: "x", temperature: 0.2 } }, offline);60 assert.equal(lint.data.valid, false);61 assert.match(lint.data.findings[0].api_error, /temperature/);62 const fix = await call("fix_request", { request: { model: "gpt-5", messages: [{ role: "user", content: "x" }], max_tokens: 50, temperature: 0.1, stop: ["a"] } }, offline);63 assert.equal(fix.data.remaining.valid, true);64 assert.equal(fix.data.request.max_completion_tokens, 50);65 const conv = await call("chat_to_responses", { request: { model: "gpt-5.4", messages: [{ role: "system", content: "Be brief." }, { role: "user", content: "hi" }], max_tokens: 100 } }, offline);66 assert.deepEqual(conv.data.request, { model: "gpt-5.4", instructions: "Be brief.", input: "hi", max_output_tokens: 100 });67 const mig = await call("migrate_request", { request: { model: "gpt-4o", messages: [{ role: "user", content: "hi" }], max_tokens: 100, temperature: 0.4 }, target_model: "gpt-5.5-pro" }, offline);68 assert.equal(mig.data.converted_to, "responses");69 assert.equal(mig.data.remaining.valid, true);70 assert.equal((await call("estimate_cost", { model: "gpt-6.1-sol", input_tokens: 200000, output_tokens: 200000 }, offline)).data.usd_total, 2.4);71 assert.equal((await call("compare_costs", { input_tokens: 1000, output_tokens: 1000 }, offline)).data.ranking[0].model, "gpt-6-luna");72 assert.equal((await call("recommend_model", { task: "classify support tickets into 5 labels", volume: "high" }, offline)).data.model, "gpt-6-luna");73 assert.equal((await call("recommend_model", { task: "autonomous multi-step refactor of a large codebase", priority: "quality" }, offline)).data.model, "gpt-6-astra");74 const rq = await call("recommend_model", { task: "write marketing copy", priority: "quality", requires: ["temperature"] }, offline);75 assert.notEqual(rq.data.model, "gpt-6.1-sol");76 assert.equal(rq.data.reasoning_effort, "none");77 assert.match((await call("scaffold_app", { approach: "function_loop", language: "python", model: "gpt-6.1-sol" }, offline)).data.code, /function_call_output/);78 const ex = await call("explain_error", { status: 400, message: "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead." }, offline);79 assert.match(ex.data.fix, /max_completion_tokens/);80 assert.match((await call("feature_guide", { topic: "sampling" }, offline)).data.guide, /none/);81 const noKey = await call("list_models", {}, offline);82 assert.equal(noKey.isError, true);83 assert.match(noKey.data.error.message, /OPENAI_API_KEY/);84});8586test("live: models and token counting", { skip }, async () => {87 const list = await call("list_models", { kind: "text" });88 assert.ok(list.data.count >= 20);89 assert.equal(list.data.models.find((m) => m.id === "gpt-6.1-sol").verified_by_probes, true);90 assert.ok((await call("list_models", { kind: "embedding" })).data.models.some((m) => m.id === "text-embedding-3-small"));91 const ct = await call("count_tokens", { request: { model: M, instructions: "You are terse.", input: "How many tokens is this sentence?" } });92 assert.ok(ct.data.input_tokens > 5 && ct.data.input_tokens < 80, String(ct.data.input_tokens));93 assert.equal(ct.data.context_window, 1050000);94});9596test("live: create_response lints, fixes, sends, and reports usage and cost", { skip }, async () => {97 const blocked = await call("create_response", { request: { model: "gpt-6.1-sol", input: "hi", temperature: 0.3 } });98 assert.equal(blocked.data.sent, false);99 assert.ok(blocked.data.lint.findings.some((f) => f.code === "sampling-param-rejected"));100 const fixed = await call("create_response", { auto_fix: true, request: { model: M, input: "Reply with exactly: pong", max_tokens: 2000, temperature: 0, reasoning_effort: "minimal" } });101 assert.equal(fixed.data.sent, true);102 assert.ok(fixed.data.applied_fixes.length >= 3);103 assert.match(fixed.data.text.toLowerCase(), /pong/);104 assert.equal(fixed.data.status, "completed");105 assert.ok(fixed.data.usage.output_tokens > 0 && fixed.data.estimated_cost_usd > 0);106 const guard = await call("create_response", { request: { model: M, input: "hi", max_output_tokens: 100000 } });107 assert.match(guard.data.error.message, /spend guard/);108 // The linter said the API would reject it; skip_lint proves the API agrees.109 const forced = await call("create_response", { skip_lint: true, request: { model: "gpt-6.1-sol", input: "hi", temperature: 0.3, max_output_tokens: 64 } });110 assert.equal(forced.isError, true);111 assert.equal(forced.data.error.status, 400);112 assert.equal(forced.data.error.param, "temperature");113 // …and what the linter allows, the API accepts: temperature with effort none.114 const allowed = await call("create_response", { request: { model: M, input: "Reply: ok", temperature: 0.2, reasoning: { effort: "none" }, max_output_tokens: 64 } });115 assert.equal(allowed.data.status, "completed");116 const chat = await call("create_response", { request: { model: "gpt-4.1-mini", messages: [{ role: "user", content: "Reply with exactly: chat-ok" }], max_completion_tokens: 20 } });117 assert.equal(chat.data.api, "chat");118 assert.match(chat.data.text, /chat-ok/);119});120121test("live: function calling round trip, stored state, image input", { skip }, async () => {122 const tools = [{ type: "function", name: "get_weather", description: "Get the current weather for a city. Always use it for weather questions.", strict: true, parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"], additionalProperties: false } }];123 const first = await call("create_response", { request: { model: M, input: "What is the weather in Paris? Use the tool.", tools, max_output_tokens: 2000 } });124 const fc = first.data.function_calls[0];125 assert.equal(fc.name, "get_weather");126 assert.match(JSON.parse(fc.arguments).city, /Paris/);127 const second = await call("create_response", { request: { model: M, previous_response_id: first.data.id, tools, input: [{ type: "function_call_output", call_id: fc.call_id, output: "Rainy, 9°C" }], max_output_tokens: 2000 } });128 assert.equal(second.data.status, "completed");129 assert.match(second.data.text, /9/);130 assert.equal((await call("get_response", { id: second.data.id })).data.text, second.data.text);131 assert.equal((await call("delete_response", { id: second.data.id })).isError, true);132 assert.equal((await call("delete_response", { id: second.data.id, confirm: true })).data.deleted, true);133 assert.equal((await call("get_response", { id: second.data.id })).isError, true);134 const img = await call("create_response", { request: { model: M, max_output_tokens: 1000, input: [{ role: "user", content: [{ type: "input_image", image_url: pngDataUrl(0, 0, 255), detail: "low" }, { type: "input_text", text: "This image is one flat colour. Is it red, green or blue? One word." }] }] } });135 assert.match(img.data.text.toLowerCase(), /blue/);136});137138test("live: structured extraction, web search, embeddings, moderation", { skip }, async () => {139 const ex = await call("extract_structured", { model: M, prompt: "Invoice 4471 from Acme Corp, dated 2026-03-02, total 1,284.50 CAD, 3 line items.", schema: { type: "object", properties: { invoice_number: { type: "string" }, vendor: { type: "string" }, total: { type: "number" }, currency: { type: "string" }, line_items: { type: "integer" } } } });140 assert.equal(ex.data.ok, true, JSON.stringify(ex.data));141 assert.deepEqual(ex.data.data, { invoice_number: "4471", vendor: "Acme Corp", total: 1284.5, currency: "CAD", line_items: 3 });142 const web = await call("create_response", { request: { model: M, tools: [{ type: "web_search" }], input: "Search the web: what is the URL of the official Model Context Protocol site? One line.", max_output_tokens: 3000 } });143 assert.ok(web.data.output.some((o) => o.type === "web_search_call"), "no web_search_call item");144 assert.match(web.data.text, /modelcontextprotocol/i);145 assert.match(web.data.cost_note, /billed separately/);146 const emb = await call("create_embeddings", { input: ["How do I reset my password?", "Steps to change a forgotten password", "Best pizza toppings"] });147 assert.equal(emb.data.dimensions, 1536);148 assert.ok(emb.data.similarity[0][1] > emb.data.similarity[0][2] + 0.2);149 assert.equal(emb.data.vectors, undefined);150 assert.equal((await call("create_embeddings", { input: ["a"], dimensions: 256, include_vectors: true })).data.vectors[0].length, 256);151 const mod = await call("moderate", { input: "I love gardening and baking bread." });152 assert.equal(mod.data.flagged, false);153});154155test("live: files and vector-store retrieval; path confinement", { skip }, async () => {156 writeFileSync(join(work, "notes.txt"), "The launch codename is OSPREY. The budget owner is Dana.");157 const up = await call("file_upload", { path: "notes.txt" });158 assert.match(up.data.id, /^file-/);159 const outside = await call("file_upload", { path: "../../../../etc/hosts" });160 assert.equal(outside.isError, true);161 assert.match(outside.data.error.message, /outside the allowed root|ENOENT/);162 assert.equal((await call("file_upload", { path: "notes.txt", content: "x" })).isError, true);163 assert.equal((await call("file_get", { id: up.data.id })).data.filename, "notes.txt");164 assert.ok((await call("file_list", { limit: 50 })).data.files.some((f) => f.id === up.data.id));165 const vs = await call("vector_store_create", { name: `mcp-test-${Date.now()}`, expires_after_days: 1 });166 assert.match(vs.data.id, /^vs_/);167 const add = await call("vector_store_add", { vector_store_id: vs.data.id, path: "notes.txt" });168 assert.equal(add.data.status, "completed", JSON.stringify(add.data));169 const hit = await call("vector_store_search", { vector_store_id: vs.data.id, query: "What is the launch codename?" });170 assert.match(hit.data.results[0].text, /OSPREY/);171 const rag = await call("create_response", { request: { model: M, tools: [vs.data.use_in_request], input: "Using file search: what is the launch codename? One word.", max_output_tokens: 3000 } });172 assert.match(rag.data.text, /OSPREY/);173 assert.ok((await call("vector_store_list", { limit: 50 })).data.vector_stores.some((v) => v.id === vs.data.id));174 assert.equal((await call("vector_store_delete", { id: vs.data.id })).isError, true);175 assert.equal((await call("vector_store_delete", { id: vs.data.id, confirm: true })).data.deleted, true);176 assert.equal((await call("file_delete", { id: up.data.id })).isError, true);177 assert.equal((await call("file_delete", { id: up.data.id, confirm: true })).data.deleted, true);178 for (const f of (await call("file_list", { limit: 100 })).data.files.filter((f) => f.filename === "notes.txt")) await call("file_delete", { id: f.id, confirm: true });179});180181test("live: speech round trip and image generation write only inside the file root", { skip }, async () => {182 const tts = await call("text_to_speech", { text: "The registry launched in November.", output_path: "speech.mp3" });183 assert.equal(tts.isError, false, JSON.stringify(tts.data));184 assert.ok(statSync(join(work, "speech.mp3")).size > 2000);185 assert.equal((await call("text_to_speech", { text: "again", output_path: "speech.mp3" })).isError, true); // never overwrites186 assert.equal((await call("text_to_speech", { text: "x", output_path: "../escape.mp3" })).isError, true);187 assert.equal(existsSync(join(work, "..", "escape.mp3")), false);188 const stt = await call("transcribe_audio", { path: "speech.mp3" });189 assert.match(stt.data.text.toLowerCase(), /registry/);190 assert.match(stt.data.text.toLowerCase(), /november/);191 const img = await call("generate_image", { prompt: "A plain solid blue square, flat colour, no text.", output_path: "square.png" });192 assert.equal(img.isError, false, JSON.stringify(img.data));193 assert.ok(statSync(join(work, "square.png")).size > 1000);194});195196test("live: batches — lint gate, create, status, cancel", { skip }, async () => {197 const bad = await call("batch_create", { requests: [{ custom_id: "a", body: { model: "gpt-6.1-sol", input: "x", temperature: 0.5 } }] });198 assert.equal(bad.data.submitted, false);199 assert.equal((await call("batch_create", { requests: [{ custom_id: "a", body: { model: M, input: "x" } }, { custom_id: "a", body: { model: M, input: "y" } }] })).isError, true);200 const b = await call("batch_create", { requests: ["two", "three"].map((w) => ({ custom_id: `q-${w}`, body: { model: M, input: `Reply with exactly the digit for the number ${w}.`, max_output_tokens: 1000 } })) });201 assert.equal(b.data.submitted, true, JSON.stringify(b.data));202 assert.ok(["validating", "in_progress", "finalizing", "completed"].includes((await call("batch_get", { id: b.data.id })).data.status));203 assert.ok((await call("batch_list", { limit: 10 })).data.batches.some((x) => x.id === b.data.id));204 const victim = await call("batch_create", { requests: [{ custom_id: "c", body: { model: M, input: "x", max_output_tokens: 100 } }] });205 assert.equal((await call("batch_cancel", { id: victim.data.id })).isError, true);206 assert.ok(["cancelling", "cancelled", "completed"].includes((await call("batch_cancel", { id: victim.data.id, confirm: true })).data.status));207 // OpenAI batches may take up to 24 h; wait up to 12 minutes and assert results only if it completed.208 let status;209 for (let i = 0; i < 48; i++) { status = (await call("batch_get", { id: b.data.id })).data; if (["completed", "failed", "expired"].includes(status.status)) break; await new Promise((r) => setTimeout(r, 15_000)); }210 if (status.status === "completed") {211 const res = await call("batch_results", { id: b.data.id });212 const byId = Object.fromEntries(res.data.results.map((r) => [r.custom_id, r]));213 assert.match(byId["q-two"].text, /2/);214 assert.match(byId["q-three"].text, /3/);215 assert.ok(res.data.estimated_cost_usd > 0);216 console.log("# batch completed and results verified");217 } else {218 assert.equal((await call("batch_results", { id: b.data.id })).data.ready, false);219 console.log(`# batch still ${status.status} after 12 minutes: results not verified in this run`);220 }221});222223test("the API key never appears in any tool output", { skip }, () => {224 const all = seen.join("\n");225 assert.ok(seen.length > 60);226 assert.equal(all.includes(KEY), false);227 assert.equal(/sk-proj-[A-Za-z0-9_-]{20,}/.test(all), false);228});229