Repository Public Research preview
1// Integration tests against the real Safari, through the MCP protocol, on a local fixture site.2// They open ONE dedicated window and close it at the end; the user's own windows are not touched.3// Run: node --test test/safari.test.mjs (macOS, Safari with “Allow JavaScript from Apple Events”)4import assert from "node:assert/strict";5import { execFileSync } from "node:child_process";6import { createHash } from "node:crypto";7import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";8import { tmpdir } from "node:os";9import { dirname, join } from "node:path";10import { after, before, test } from "node:test";11import { fileURLToPath } from "node:url";12import { Client } from "@modelcontextprotocol/sdk/client/index.js";13import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";14import { REPORT_SHA, startFixtures } from "./fixtures/server.mjs";1516const root = dirname(dirname(fileURLToPath(import.meta.url)));17const work = mkdtempSync(join(tmpdir(), "safari-mcp-"));18const skip = process.platform !== "darwin" && "macOS only";19let fx, c, userWindowsBefore;2021/** Window ids and tab counts only: never URLs or titles of the user's own tabs. */22function windowCensus() {23 const out = execFileSync("/usr/bin/osascript", ["-e", 'set o to ""\ntell application "Safari"\nrepeat with w in windows\nif visible of w and (count of tabs of w) > 0 then set o to o & (id of w) & ":" & (count of tabs of w) & " "\nend repeat\nend tell\nreturn o'], { encoding: "utf8" });24 return out.trim().split(/\s+/).filter(Boolean).sort();25}26async function connect(env = {}) {27 const cl = new Client({ name: "safari-test", version: "1" });28 await cl.connect(new StdioClientTransport({ command: process.execPath, args: [join(root, "server.mjs")], env: { PATH: process.env.PATH, SAFARI_MCP_FILE_ROOT: work, ...env }, stderr: "pipe" }));29 return cl;30}31async function call(name, args = {}, client = c) {32 const r = await client.callTool({ name, arguments: args }, undefined, { timeout: 120_000 });33 return { data: r.structuredContent, isError: !!r.isError, content: r.content };34}35const text = async (selector) => (await call("read_text", { target: { selector } })).data.text;3637before(async () => {38 if (skip) return;39 userWindowsBefore = windowCensus();40 fx = await startFixtures();41 c = await connect();42});43after(async () => {44 if (skip) return;45 await call("close_tab", { window: true }).catch(() => {});46 await c?.close();47 await fx?.close();48 rmSync(work, { recursive: true, force: true });49});5051test("status reports a drivable Safari", { skip }, async () => {52 const s = await call("safari_status");53 assert.equal(s.data.running, true);54 assert.equal(s.data.javascript_from_apple_events, true, "enable “Allow JavaScript from Apple Events” in Safari");55 assert.equal(s.data.scope, "owned");56 assert.equal(s.data.evaluate_js_enabled, false);57 assert.equal(s.data.owned_window, null);58 assert.equal((await call("snapshot")).data.error.code, "no_window");59});6061test("opens a page in its own window and snapshots it", { skip }, async () => {62 const o = await call("open_url", { url: fx.origin + "/" });63 assert.equal(o.data.tab, "t1");64 assert.equal(o.data.title, "Fixture home");65 assert.equal(o.data.loaded, true);66 const snap = await call("snapshot");67 const names = snap.data.elements.map((e) => `${e.role}:${e.name}`);68 for (const n of ["link:Open the form", "link:Price table", "button:Reveal later", "button:Shadow button", "button:Frame button"]) assert.ok(names.includes(n), n);69 assert.deepEqual(snap.data.headings[0], { level: 1, text: "Fixture home" });70 assert.deepEqual(snap.data.cross_origin_frames, [`http://localhost:${fx.port}/frame.html`]);71 assert.match(snap.data._notice, /untrusted/);72 assert.ok(snap.data.elements.every((e) => /^e\d+$/.test(e.ref)));73 const tabs = await call("list_tabs");74 assert.equal(tabs.data.tabs.length, 1);75 assert.equal(tabs.data.tabs[0].current, true);76});7778test("the user's own tabs are out of reach by default", { skip }, async () => {79 assert.equal((await call("list_tabs", { all: true })).data.error.code, "tab_out_of_scope");80 const foreign = userWindowsBefore[0]?.split(":")[0];81 if (foreign) {82 for (const [tool, args] of [["read_text", {}], ["snapshot", {}], ["click", { target: { selector: "a" } }], ["close_tab", {}], ["screenshot", { output_path: "x.png" }]]) {83 const r = await call(tool, { tab: `w${foreign}:t1`, ...args });84 assert.equal(r.data.error.code, "tab_out_of_scope", tool);85 }86 }87});8889test("reads text, links, tables, forms and elements", { skip }, async () => {90 const t = await call("read_text");91 assert.match(t.data.text, /Delayed content/);92 assert.ok(!t.data.text.includes("This paragraph is hidden")); // innerText: visible text only93 assert.equal((await call("read_text", { target: { selector: "h1" } })).data.text, "Fixture home");94 const links = await call("list_links", { filter: "form" });95 assert.deepEqual(links.data.links, [{ text: "Open the form", href: fx.origin + "/form.html" }]);96 assert.equal((await call("query_elements", { selector: "button.dup", attributes: ["class"] })).data.count, 2);97 assert.match((await call("read_html", { target: { selector: "nav" }, max_chars: 500 })).data.html, /^<nav>/);98 await call("navigate", { url: fx.origin + "/table.html" });99 const tb = await call("extract_tables", { max_rows: 5 });100 assert.equal(tb.data.tables[0].caption, "Quarterly prices");101 assert.deepEqual(tb.data.tables[0].headers, ["Item", "Quarter", "Price"]);102 assert.deepEqual(tb.data.tables[0].rows[2], ["Item 3", "Q3", "30.50"]);103 assert.equal(tb.data.tables[0].total_rows, 30);104 assert.equal(tb.data.tables[0].truncated, true);105 const paged = await call("read_text", { max_chars: 100, offset: 50 });106 assert.equal(paged.data.text.length, 100);107 assert.equal(paged.data.truncated, true);108});109110test("clicks through shadow DOM and same-origin frames, reports ambiguity, waits for late content", { skip }, async () => {111 await call("navigate", { url: fx.origin + "/" });112 const sh = await call("click", { target: { text: "Shadow button" } });113 assert.equal(sh.data.page.title, "Fixture home (shadow clicked)");114 assert.equal(sh.data.page.navigated, false);115 const fr = await call("click", { target: { text: "Frame button" } });116 assert.equal(fr.data.clicked.in_frame, true);117 assert.equal(await text("#frame-out"), "frame clicked");118 const amb = await call("click", { target: { text: "Save" } });119 assert.equal(amb.isError, true);120 assert.match(amb.data.error.message, /ambiguous: 2 visible matches/);121 assert.equal(amb.data.error.candidates.length, 2);122 assert.equal((await call("click", { target: { text: "Save", index: 1 } })).isError, false);123 assert.match((await call("click", { target: { selector: "#does-not-exist" } })).data.error.message, /no element matches/);124 assert.match((await call("click", { target: { selector: "#hidden-note" } })).data.error.message, /none is visible/);125 assert.match((await call("click", { target: { selector: "a[[" } })).data.error.message, /invalid selector/);126 await call("click", { target: { selector: "#reveal" } });127 const gone = await call("wait_for", { selector: "#arrived", timeout_seconds: 0.2 });128 assert.equal(gone.data.met, false); // not there yet: reported, not thrown129 const w = await call("wait_for", { selector: "#arrived", text: "The content arrived", timeout_seconds: 5 });130 assert.equal(w.data.met, true);131 assert.ok(w.data.waited_ms < 4500);132 assert.equal((await call("wait_for", {})).data.error.code, "condition_missing");133});134135test("page dialogs are recorded and answered instead of freezing the browser", { skip }, async () => {136 await call("click", { target: { selector: "#ask" } });137 assert.equal(await text("#answer"), "confirmed");138 const info = await call("page_info");139 // The dialog was reported by the click's own settle step or is reported now; either way the page did not block.140 assert.equal(info.data.ready_state, "complete");141});142143test("follows links, redirects, history and new tabs", { skip }, async () => {144 const nav = await call("click", { target: { text: "Open the form", role: "link" } });145 assert.equal(nav.data.page.navigated, true);146 assert.equal(nav.data.page.title, "Fixture form");147 assert.equal((await call("navigate", { action: "back" })).data.title.startsWith("Fixture home"), true);148 assert.equal((await call("navigate", { action: "forward" })).data.title, "Fixture form");149 const red = await call("navigate", { url: fx.origin + "/redirect" });150 assert.equal(red.data.url, fx.origin + "/about.html"); // final URL after the redirect151 const slow = await call("open_url", { url: fx.origin + "/slow.html" });152 assert.equal(slow.data.tab, "t2");153 assert.equal(slow.data.title, "Slow page");154 assert.ok(slow.data.waited_ms >= 1000);155 const tabs = await call("list_tabs");156 assert.deepEqual(tabs.data.tabs.map((t) => [t.tab, t.current]), [["t1", false], ["t2", true]]);157 assert.equal((await call("read_text", { tab: "t1", target: { selector: "h1" } })).data.text, "About"); // background tab stays scriptable158 assert.equal((await call("select_tab", { tab: "t1" })).data.title, "About the fixture");159 assert.equal((await call("close_tab", { tab: "t2" })).data.remaining_tabs, 1);160 assert.equal((await call("navigate", { url: fx.origin + "/missing.html" })).data.title, "Not found");161 assert.equal((await call("navigate", { url: "javascript:alert(1)" })).data.error.code, "url_forbidden");162 assert.equal((await call("navigate", {})).data.error.code, "url_missing");163});164165test("fills a framework-style form, guards secrets, uploads a file and submits", { skip }, async () => {166 await call("navigate", { url: fx.origin + "/form.html" });167 const forms = await call("list_forms");168 assert.equal(forms.data.forms[0].method, "post");169 assert.ok(forms.data.forms[0].fields.some((f) => f.name === "Password" && f.sensitive === true));170171 await call("fill", { target: { label: "Full name" }, value: "Ada Lovelace" });172 assert.equal(await text("#mirror"), "Ada Lovelace"); // the page's own input listener saw the change173174 const pw = await call("fill", { target: { label: "Password" }, value: "hunter2-not-real" });175 assert.equal(pw.data.error.code, "sensitive_field");176 assert.equal((await call("fill", { target: { label: "Password" }, value: "fixture-only-password", confirm_sensitive: true })).isError, false);177 const snap = await call("snapshot");178 assert.equal(snap.data.elements.find((e) => e.name === "Password").value, "[redacted]");179 assert.ok(!JSON.stringify(snap.data).includes("fixture-only-password"));180 assert.ok(!JSON.stringify((await call("list_forms")).data).includes("fixture-only-password"));181182 const batch = await call("fill_form", { fields: [{ target: { label: "Email" }, value: "ada@example.org" }, { target: { label: "Notes" }, value: "Line one" }, { target: { label: "I accept the terms" }, checked: true }, { target: { selector: 'input[name="plan"][value="pro"]' }, checked: true }, { target: { label: "Biography" }, value: "Mathematician." }] });183 assert.equal(batch.data.completed, true);184 assert.equal(batch.data.filled, 5);185 const partial = await call("fill_form", { fields: [{ target: { label: "Notes" }, value: "ok" }, { target: { label: "No such field" }, value: "x" }] });186 assert.equal(partial.data.completed, false);187 assert.equal(partial.data.failed_index, 1);188 assert.equal((await call("select_option", { target: { label: "Country" }, value: "Japan" })).data.value, "jp");189 assert.match((await call("select_option", { target: { label: "Country" }, value: "Atlantis" })).data.error.message, /Options: .*Canada/);190 assert.equal((await call("set_checked", { target: { label: "I accept the terms" }, checked: true })).data.changed, false);191 assert.equal((await call("fill", { target: { label: "I accept the terms" }, value: "x" })).isError, true);192193 writeFileSync(join(work, "cv.txt"), "curriculum vitae fixture\n");194 const up = await call("upload_file", { target: { label: "Document" }, path: "cv.txt" });195 assert.deepEqual(up.data.files, [{ name: "cv.txt", size: 25, type: "text/plain" }]);196 await call("wait_for", { text: "cv.txt 25", timeout_seconds: 5 });197 assert.equal(await text("#doc-info"), `cv.txt 25 ${createHash("sha256").update("curriculum vitae fixture\n").digest("hex")}`); // bytes arrive intact198 assert.equal((await call("upload_file", { target: { label: "Document" }, path: "../../etc/hosts" })).isError, true);199 assert.equal((await call("upload_file", { target: { label: "Full name" }, path: "cv.txt" })).isError, true);200201 const sub = await call("click", { target: { text: "Create account" } });202 assert.equal(sub.data.page.navigated, true);203 assert.equal(sub.data.page.title, "Account created");204 const got = JSON.parse(await text("#received"));205 assert.deepEqual({ name: got.name, email: got.email, country: got.country, terms: got.terms, plan: got.plan, notes: got.notes, bio: got.bio, doc: got.doc }, { name: "Ada Lovelace", email: "ada@example.org", country: "jp", terms: "yes", plan: "pro", notes: "ok", bio: "Mathematician.", doc: "file:cv.txt:25" });206});207208test("keyboard submit, scrolling and hover", { skip }, async () => {209 await call("navigate", { url: fx.origin + "/form.html" });210 await call("fill", { target: { label: "Search the site" }, value: "tide tables" });211 const k = await call("press_key", { key: "Enter", target: { label: "Search the site" } });212 assert.equal(k.data.form_submitted, true);213 assert.match(k.data.page.url, /\/search\?q=tide\+tables$/);214 assert.equal(await text("#q"), "tide tables");215 await call("navigate", { url: fx.origin + "/long.html" });216 const down = await call("scroll", {});217 assert.ok(down.data.scroll.y > 200);218 const bottom = await call("scroll", { to: "bottom" });219 assert.equal(bottom.data.scroll.y, bottom.data.scroll.max_y);220 assert.equal((await call("scroll", { to: "top" })).data.scroll.y, 0);221 await call("click", { target: { selector: "#bottom-btn" } }); // off-screen element: scrolled into view first222 assert.equal(await text("#bottom-out"), "bottom clicked");223 assert.equal((await call("hover", { target: { selector: "#bottom-btn" } })).data.hovered.name, "Bottom button");224 const vp = await call("snapshot", { viewport_only: true });225 assert.ok(vp.data.elements.some((e) => e.name === "Bottom button"));226});227228test("downloads and fetches with the page's signed-in session", { skip }, async () => {229 await call("navigate", { url: fx.origin + "/" });230 const denied = await call("download", { url: "/download/report.bin", output_path: "report.bin" });231 assert.equal(denied.data.error.code, "http_error"); // not signed in yet: HTTP 403, nothing saved232 assert.equal(existsSync(join(work, "report.bin")), false);233 await call("navigate", { url: fx.origin + "/login" }); // sets an HttpOnly cookie the server never sees234 const dl = await call("download", { url: "/download/report.bin", output_path: "files/report.bin" });235 assert.equal(dl.data.bytes, 300000);236 assert.equal(createHash("sha256").update(readFileSync(join(work, "files/report.bin"))).digest("hex"), REPORT_SHA);237 assert.equal((await call("download", { url: "/download/report.bin", output_path: "files/report.bin" })).data.error.code, "file_exists");238 assert.equal((await call("download", { url: "/download/report.bin", output_path: "../escape.bin" })).data.error.code, "path_forbidden");239 const api = await call("fetch_text", { url: "/api/data.json" });240 assert.deepEqual(JSON.parse(api.data.text), { items: [1, 2, 3], signed_in: true });241 const cross = await call("fetch_text", { url: `http://localhost:${fx.port}/api/data.json` });242 assert.match(cross.data.error.message, /cross-origin fetch refused/);243});244245test("screenshot writes a PNG inside the file root and returns a preview", { skip }, async () => {246 await call("navigate", { url: fx.origin + "/table.html" });247 await call("resize_window", { width: 900, height: 700 });248 const s = await call("screenshot", { output_path: "shots/table.png" });249 assert.equal(s.isError, false, JSON.stringify(s.data));250 assert.ok(statSync(join(work, "shots/table.png")).size > 5000);251 assert.deepEqual([...readFileSync(join(work, "shots/table.png")).subarray(0, 4)], [0x89, 0x50, 0x4e, 0x47]);252 assert.equal(s.content.find((x) => x.type === "image")?.mimeType, "image/jpeg");253 assert.equal((await call("screenshot", { output_path: "shots/table.png" })).data.error.code, "file_exists");254 assert.equal((await call("screenshot", { output_path: "/tmp/outside.png" })).data.error.code, "path_forbidden");255});256257test("evaluate_js is off unless the operator enables it; domain policy is enforced live", { skip }, async () => {258 assert.equal((await call("evaluate_js", { code: "document.title" })).data.error.code, "eval_disabled");259 const strict = await connect({ SAFARI_MCP_ALLOW_EVAL: "1", SAFARI_MCP_ALLOW_DOMAINS: "127.0.0.1", SAFARI_MCP_BLOCK_DOMAINS: "localhost" });260 try {261 assert.equal((await call("open_url", { url: `http://localhost:${fx.port}/` }, strict)).data.error.code, "domain_blocked");262 assert.equal((await call("open_url", { url: "https://example.org/" }, strict)).data.error.code, "domain_not_allowed");263 assert.equal((await call("open_url", { url: fx.origin + "/table.html" }, strict)).data.title, "Price table");264 assert.equal((await call("evaluate_js", { code: "document.querySelectorAll('tbody tr').length" }, strict)).data.result, 30);265 assert.match((await call("evaluate_js", { code: "nope.nope" }, strict)).data.error.message, /nope/);266 // A page that moves the tab to a host outside the policy is refused on the next call.267 await call("evaluate_js", { code: `location.href = "http://localhost:${fx.port}/about.html"; 1` }, strict);268 await new Promise((r) => setTimeout(r, 1200));269 assert.equal((await call("read_text", {}, strict)).data.error.code, "domain_blocked");270 } finally { await call("close_tab", { window: true }, strict); await strict.close(); }271});272273test("stale refs are reported, and the session window closes cleanly", { skip }, async () => {274 await call("navigate", { url: fx.origin + "/" });275 const ref = (await call("snapshot")).data.elements.find((e) => e.name === "Price table").ref;276 await call("navigate", { action: "reload" });277 assert.match((await call("click", { target: { ref } })).data.error.message, /stale/);278 assert.equal((await call("close_tab", { window: true })).data.closed, true);279 assert.equal((await call("read_text")).data.error.code, "no_window");280 assert.equal((await call("close_tab", { window: true })).data.closed, false);281});282283test("the user's windows are exactly as they were", { skip }, async () => {284 await new Promise((r) => setTimeout(r, 500));285 assert.deepEqual(windowCensus(), userWindowsBefore);286});287