Repository Public Research preview
1// Tests of the skill against the real Safari: every AppleScript recipe and the command-line tool,2// on the local fixture site, in dedicated windows that are closed afterwards.3// Run: node --test skill-tests/skill.test.mjs4import assert from "node:assert/strict";5import { execFile, execFileSync } from "node:child_process";6import { createHash } from "node:crypto";7import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";8import { tmpdir } from "node:os";9import { join } from "node:path";10import { after, before, test } from "node:test";11import { fileURLToPath } from "node:url";12import { REPORT_SHA, startFixtures } from "../mcp/test/fixtures/server.mjs";1314const skill = fileURLToPath(new URL("../skill/safari-applescript/", import.meta.url));15const recipes = join(skill, "scripts/recipes");16const work = mkdtempSync(join(tmpdir(), "safari-skill-"));17const state = join(work, "state.json");18// Asynchronous on purpose: the fixture server lives in this process, and a synchronous child would freeze it.19const run = (cmd, argv, opts = {}) => new Promise((res) => execFile(cmd, argv, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...opts }, (err, stdout, stderr) => res({ status: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout, stderr })));20const osa = async (file, ...args) => (await run("/usr/bin/osascript", [join(recipes, file), ...args.map(String)])).stdout.replace(/\n$/, "");21const cli = async (...args) => { const r = await run(process.execPath, [join(skill, "scripts/safari.mjs"), ...args], { cwd: work, env: { PATH: process.env.PATH, SAFARI_SKILL_STATE: state, SAFARI_MCP_FILE_ROOT: work } }); let json = null; try { json = JSON.parse(r.stdout); } catch { /* usage text */ } return { status: r.status, json, stderr: r.stderr }; };22const census = () => execFileSync("/usr/bin/osascript", ["-e", 'set o to ""\ntell application "Safari"\nrepeat with w in windows\nif visible of w and (count of tabs of w) > 0 then set o to o & (id of w) & ":" & (count of tabs of w) & " "\nend repeat\nend tell\nreturn o'], { encoding: "utf8" }).trim().split(/\s+/).filter(Boolean).sort();23let fx, before0, win;2425before(async () => { before0 = census(); fx = await startFixtures(); });26after(async () => {27 if (win) await osa("close-window.applescript", win).catch(() => {});28 (await cli("close", "--window"));29 await fx.close();30 rmSync(work, { recursive: true, force: true });31});3233test("SKILL.md follows the Agent Skills specification and references exist", () => {34 const md = readFileSync(join(skill, "SKILL.md"), "utf8");35 const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];36 assert.equal(/^name: (.+)$/m.exec(fm)[1], "safari-applescript");37 const d = /^description: (.+)$/m.exec(fm)[1];38 assert.ok(d.length > 50 && d.length <= 1024);39 assert.ok(md.split("\n").length < 500);40 for (const ref of md.matchAll(/`((?:references|scripts)\/[A-Za-z0-9_./-]+)`/g)) assert.ok(existsSync(join(skill, ref[1])), `${ref[1]} is referenced but missing`);41 for (const f of readdirSync(recipes)) assert.ok(md.includes(f) || readFileSync(join(skill, "references/applescript-recipes.md"), "utf8").includes(f), `${f} is not documented`);42});4344test("the skill engine is byte-identical to the MCP engine and has no dependencies", () => {45 for (const f of ["osa.mjs", "safari.mjs", "page.js"]) assert.equal(readFileSync(join(skill, "scripts/lib", f), "utf8"), readFileSync(new URL(`../mcp/lib/${f}`, import.meta.url), "utf8"), f);46 for (const f of ["scripts/safari.mjs", "scripts/lib/osa.mjs", "scripts/lib/safari.mjs"]) for (const m of readFileSync(join(skill, f), "utf8").matchAll(/^import .* from "([^"]+)";$/gm)) assert.ok(m[1].startsWith("node:") || m[1].startsWith("./"), `${f} imports ${m[1]}`);47});4849test("recipes: open a window, wait for load, list tabs, read text", async () => {50 win = Number((await osa("open-window.applescript", fx.origin + "/slow.html")));51 assert.ok(win > 0);52 assert.equal((await osa("wait-for-load.applescript", win, 1, 20)), "loaded");53 assert.equal((await osa("list-tabs.applescript", win)).trim(), `1\t${fx.origin}/slow.html\tSlow page`); // not the blank page it started on54 assert.match((await osa("get-text.applescript", win, 1)), /Served after a delay/);55 assert.equal((await osa("run-js.applescript", win, 1, "JSON.stringify({t: document.title, n: 1 + 1})")), '{"t":"Slow page","n":2}');56 assert.equal((await osa("run-js.applescript", win, 1, "undefined")), ""); // undefined comes back empty, not as an error57});5859test("recipes: fill a framework-style field safely, click, wait for the navigation", async () => {60 (await osa("run-js.applescript", win, 1, `location.href = ${JSON.stringify(fx.origin + "/form.html")}; ""`));61 assert.equal((await osa("wait-for-load.applescript", win, 1, 20)), "loaded");62 const tricky = `O'Brien "quoted" \\ back\\slash'); document.title='pwned'; ('`;63 assert.equal((await osa("fill-field.applescript", win, 1, "#name", tricky)), "ok");64 assert.equal((await osa("run-js.applescript", win, 1, "document.getElementById('mirror').textContent")), tricky); // the page's listener saw the exact value65 assert.equal((await osa("run-js.applescript", win, 1, "document.title")), "Fixture form"); // and nothing was injected66 assert.equal((await osa("fill-field.applescript", win, 1, "#nope", "x")), "not-found");67 assert.equal((await osa("fill-field.applescript", win, 1, "#name", "Ada Lovelace")), "ok");68 assert.equal((await osa("fill-field.applescript", win, 1, "#country", "fr")), "ok");69 assert.equal((await osa("click.applescript", win, 1, "#send")), "ok");70 assert.equal((await osa("wait-for-load.applescript", win, 1, 20)), "loaded");71 assert.equal(JSON.parse((await osa("run-js.applescript", win, 1, "document.getElementById('received').textContent"))).name, "Ada Lovelace");72 assert.equal((await osa("click.applescript", win, 1, "#nope")), "not-found");73});7475test("recipes: download with the page's session, then close the window", async () => {76 (await osa("run-js.applescript", win, 1, `location.href = ${JSON.stringify(fx.origin + "/")}; ""`));77 (await osa("wait-for-load.applescript", win, 1, 20));78 assert.equal((await osa("fetch-base64.applescript", win, 1, "/download/report.bin")), "HTTP 403");79 (await osa("run-js.applescript", win, 1, `location.href = "/login"; ""`));80 (await osa("wait-for-load.applescript", win, 1, 20));81 const b64 = (await osa("fetch-base64.applescript", win, 1, "/download/report.bin"));82 assert.equal(createHash("sha256").update(Buffer.from(b64, "base64")).digest("hex"), REPORT_SHA);83 (await osa("close-window.applescript", win));84 win = null;85});8687test("command-line tool: full flow with state kept between invocations", async () => {88 assert.equal((await cli("snapshot")).json.error.code, "no_window");89 const o = (await cli("open", fx.origin + "/"));90 assert.equal(o.json.title, "Fixture home");91 assert.ok(JSON.parse(readFileSync(state, "utf8")).window > 0);92 const snap = (await cli("snapshot")).json; // a separate process finds the same window again93 const ref = snap.elements.find((e) => e.name === "Open the form").ref;94 const nav = (await cli("click", "--ref", ref)).json; // refs live in the page, so they survive between invocations95 assert.equal(nav.page.title, "Fixture form");96 assert.equal((await cli("fill", "--label", "Full name", "--value", "Grace Hopper")).status, 0);97 assert.equal((await cli("text", "--selector", "#mirror")).json.text, "Grace Hopper");98 assert.equal((await cli("fill", "--label", "Password", "--value", "x")).json.error.code, "sensitive_field");99 assert.equal((await cli("select", "--label", "Country", "--value", "Canada")).json.value, "ca");100 assert.equal((await cli("check", "--label", "I accept the terms")).json.changed, true);101 writeFileSync(join(work, "cv.txt"), "cv\n");102 assert.equal((await cli("upload", "--label", "Document", "--file", "cv.txt")).json.files[0].size, 3);103 assert.equal((await cli("click", "--text", "Create account")).json.page.title, "Account created");104 assert.equal((await cli("back")).json.title, "Fixture form");105 assert.equal((await cli("goto", fx.origin + "/table.html")).json.title, "Price table");106 assert.deepEqual((await cli("tables", "--rows", "2")).json.tables[0].rows[1], ["Item 2", "Q2", "20.50"]);107 assert.equal((await cli("links")).json.count, 0);108 assert.equal((await cli("query", "tbody tr")).json.count, 30);109 assert.equal((await cli("wait", "--text", "Item 30", "--timeout", "3")).status, 0);110 assert.equal((await cli("wait", "--selector", "#never", "--timeout", "0.5")).status, 2);111 assert.equal((await cli("goto", fx.origin + "/login")).json.title, "Signed in");112 assert.equal((await cli("download", "/download/report.bin", "--out", "r.bin")).json.bytes, 300000);113 assert.deepEqual(JSON.parse((await cli("fetch", "/api/data.json")).json.text), { items: [1, 2, 3], signed_in: true });114 assert.ok((await cli("screenshot", "--out", "s.png")).json.bytes > 5000);115 assert.ok(statSync(join(work, "s.png")).size > 5000);116 assert.equal((await cli("eval", "1+1")).json.error.code, "eval_disabled");117 assert.equal((await cli("open", "javascript:alert(1)")).json.error.code, "url_forbidden");118 assert.equal((await cli("nonsense")).status, 1);119 assert.equal((await cli("click")).json.error.code, "usage");120 assert.equal((await cli("close", "--window")).json.closed, true);121 assert.equal(existsSync(state), false);122});123124test("the user's windows are exactly as they were", async () => {125 await new Promise((r) => setTimeout(r, 500));126 assert.deepEqual(census(), before0);127});128