spb/llm-api-toolkits

1// Executes every scaffold template against the live API. Usage:2//   ANTHROPIC_API_KEY=… PYTHON=/path/to/python node test/run-scaffolds.mjs [approach,approach] [model]3import { execFile } from "node:child_process";4import { mkdtempSync, writeFileSync, rmSync, symlinkSync } from "node:fs";5import { join, dirname } from "node:path";6import { fileURLToPath } from "node:url";7import { APPROACHES, scaffold } from "../lib/scaffold.mjs";89const root = dirname(dirname(fileURLToPath(import.meta.url)));10const only = process.argv[2] && process.argv[2] !== "all" ? process.argv[2].split(",") : Object.keys(APPROACHES);11const model = process.argv[3] ?? "claude-haiku-5-5";12const python = process.env.PYTHON ?? "python3";13const dir = mkdtempSync(join(root, ".scaffold-"));14symlinkSync(join(root, "node_modules"), join(dir, "node_modules"));15const run = (cmd, args) => new Promise((res) => execFile(cmd, args, { cwd: dir, timeout: 900_000, env: process.env, maxBuffer: 1e7 }, (err, stdout, stderr) => res({ code: err ? err.code ?? 1 : 0, stdout, stderr })));1617const results = await Promise.all(only.flatMap((approach) => ["typescript", "python"].map(async (language) => {18  const s = scaffold({ approach, language, model });19  writeFileSync(join(dir, s.filename), s.code);20  const r = language === "typescript" ? await run(join(root, "node_modules/.bin/tsx"), [s.filename]) : await run(python, [s.filename]);21  const out = r.stdout.trim().replace(/\s+/g, " ");22  return { name: `${approach}/${language}`, ok: r.code === 0 && out.length > 0, out: out.slice(0, 110), err: r.code === 0 ? "" : r.stderr.trim().split("\n").slice(-3).join(" | ").slice(0, 400) };23})));24rmSync(dir, { recursive: true, force: true });25for (const r of results) console.log(r.ok ? "PASS" : "FAIL", r.name.padEnd(30), r.out, r.err);26process.exit(results.every((r) => r.ok) ? 0 : 1);27