spb/note-spboucher-ai

1/**2 * Shared helpers for Phase Zero (provider discovery + benchmarks).3 * Secrets are read from .env.secrets (symlink to the secrets dir, git-ignored).4 */5import fs from "node:fs";6import path from "node:path";7import OpenAI from "openai";8import Anthropic from "@anthropic-ai/sdk";910export const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");11export const RESULTS = path.join(ROOT, "benchmarks/results");12export const RAW = path.join(RESULTS, "raw");13fs.mkdirSync(RAW, { recursive: true });1415export function loadSecrets() {16  const f = path.join(ROOT, ".env.secrets");17  if (!fs.existsSync(f)) throw new Error(".env.secrets missing");18  for (const line of fs.readFileSync(f, "utf8").split("\n")) {19    const m = line.match(/^([A-Z_]+)=(.*)$/);20    if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();21  }22  if (!process.env.OPENAI_API_KEY) throw new Error("OPENAI_API_KEY missing");23  if (!process.env.ANTHROPIC_API_KEY) throw new Error("ANTHROPIC_API_KEY missing");24}25loadSecrets();2627export const openai = new OpenAI({ maxRetries: 2, timeout: 10 * 60_000 });28export const anthropic = new Anthropic({ maxRetries: 2, timeout: 10 * 60_000 });2930export type Provider = "openai" | "anthropic";3132/** Pricing in USD per 1M tokens. Source + fetch date recorded for the report. */33export const PRICING_SOURCES = {34  openai: { url: "https://developers.openai.com/api/docs/pricing", fetched: "2026-11-01", note: "Standard tier, short context (≤272K input)" },35  anthropic: { url: "https://platform.claude.com/docs/en/about-claude/pricing", fetched: "2026-11-01", note: "First-party API rates" },36};37export const PRICING: Record<string, { input: number; cached: number | null; output: number }> = {38  // OpenAI39  "gpt-6-astra": { input: 10, cached: 1, output: 50 },40  "gpt-6.1-sol": { input: 2, cached: 0.1, output: 10 },41  "gpt-6.1-terra": { input: 1, cached: 0.05, output: 5 },42  "gpt-6-luna": { input: 0.1, cached: 0.01, output: 0.5 },43  "gpt-6-sol": { input: 2, cached: 0.2, output: 10 },44  "gpt-5.6-sol": { input: 4, cached: 0.4, output: 20 },45  "gpt-5.6-terra": { input: 2, cached: 0.2, output: 12 },46  "gpt-5.6-luna": { input: 0.2, cached: 0.02, output: 1.2 },47  "gpt-5.5": { input: 5, cached: 0.5, output: 30 },48  "gpt-5.5-pro": { input: 30, cached: null, output: 180 },49  "gpt-5.4": { input: 2.5, cached: 0.25, output: 15 },50  "gpt-5.4-pro": { input: 30, cached: null, output: 180 },51  "gpt-5.4-mini": { input: 0.75, cached: 0.075, output: 4.5 },52  "gpt-5.4-nano": { input: 0.2, cached: 0.02, output: 1.25 },53  "gpt-5.3-chat-latest": { input: 1.75, cached: 0.175, output: 14 },54  "gpt-5.3-codex": { input: 1.75, cached: 0.175, output: 14 },55  "gpt-5.2": { input: 1.75, cached: 0.175, output: 14 },56  "gpt-5.2-chat-latest": { input: 1.75, cached: 0.175, output: 14 },57  "gpt-5.2-codex": { input: 1.75, cached: 0.175, output: 14 },58  "gpt-5.2-pro": { input: 21, cached: null, output: 168 },59  "gpt-5.1": { input: 1.25, cached: 0.125, output: 10 },60  "gpt-5.1-chat-latest": { input: 1.25, cached: 0.125, output: 10 },61  "gpt-5.1-codex": { input: 1.25, cached: 0.125, output: 10 },62  "gpt-5.1-codex-max": { input: 1.25, cached: 0.125, output: 10 },63  "gpt-5.1-codex-mini": { input: 0.25, cached: 0.025, output: 2 },64  "gpt-5": { input: 1.25, cached: 0.125, output: 10 },65  "gpt-5-chat-latest": { input: 1.25, cached: 0.125, output: 10 },66  "gpt-5-codex": { input: 1.25, cached: 0.125, output: 10 },67  "gpt-5-mini": { input: 0.25, cached: 0.025, output: 2 },68  "gpt-5-nano": { input: 0.05, cached: 0.005, output: 0.4 },69  "gpt-5-pro": { input: 15, cached: null, output: 120 },70  "o1": { input: 15, cached: 7.5, output: 60 },71  "o1-pro": { input: 150, cached: null, output: 600 },72  "o3-pro": { input: 20, cached: null, output: 80 },73  "o3": { input: 2, cached: 0.5, output: 8 },74  "o4-mini": { input: 1.1, cached: 0.275, output: 4.4 },75  "o3-mini": { input: 1.1, cached: 0.55, output: 4.4 },76  "gpt-4.1": { input: 2, cached: 0.5, output: 8 },77  "gpt-4.1-mini": { input: 0.4, cached: 0.1, output: 1.6 },78  "gpt-4.1-nano": { input: 0.1, cached: 0.025, output: 0.4 },79  "gpt-4o": { input: 2.5, cached: 1.25, output: 10 },80  "gpt-4o-mini": { input: 0.15, cached: 0.075, output: 0.6 },81  "chat-latest": { input: 5, cached: 0.5, output: 30 }, // alias; priced as gpt-5.5 (assumption, flagged in report)82  "text-embedding-3-small": { input: 0.02, cached: null, output: 0 },83  "text-embedding-3-large": { input: 0.13, cached: null, output: 0 },84  // Anthropic85  "claude-fable-5-1": { input: 10, cached: 0.25, output: 50 },86  "claude-fable-5": { input: 10, cached: 1, output: 50 },87  "claude-opus-5-5": { input: 4, cached: 0.2, output: 20 },88  "claude-opus-5": { input: 5, cached: 0.5, output: 25 },89  "claude-opus-4-8": { input: 5, cached: 0.5, output: 25 },90  "claude-opus-4-7": { input: 5, cached: 0.5, output: 25 },91  "claude-opus-4-6": { input: 5, cached: 0.5, output: 25 },92  "claude-opus-4-5-20251101": { input: 5, cached: 0.5, output: 25 },93  "claude-sonnet-5-5": { input: 2, cached: 0.1, output: 10 },94  "claude-sonnet-5": { input: 2, cached: 0.2, output: 10 },95  "claude-sonnet-4-6": { input: 3, cached: 0.3, output: 15 },96  "claude-sonnet-4-5-20250929": { input: 3, cached: 0.3, output: 15 },97  "claude-haiku-5-5": { input: 0.1, cached: 0.01, output: 0.5 },98  "claude-haiku-4-5-20251001": { input: 1, cached: 0.1, output: 5 },99};100101export function priceFor(model: string) {102  if (PRICING[model]) return PRICING[model];103  // dated snapshot → alias104  const alias = model.replace(/-\d{4}-\d{2}-\d{2}$/, "");105  return PRICING[alias] ?? null;106}107108export function estimateCost(model: string, inTok: number, outTok: number, cachedTok = 0) {109  const p = priceFor(model);110  if (!p) return null;111  const uncached = Math.max(0, inTok - cachedTok);112  return (uncached * p.input + cachedTok * (p.cached ?? p.input) + outTok * p.output) / 1e6;113}114115export interface Usage { input_tokens: number; output_tokens: number; cached_tokens?: number; reasoning_tokens?: number; source: "provider" }116117export interface CallResult {118  ok: boolean;119  text: string;120  toolCalls: { name: string; args: unknown }[];121  usage: Usage | null;122  ttft_ms: number | null;123  total_ms: number;124  error?: string;125  errorType?: string;126  status?: number;127  stop?: string | null;128  raw?: unknown;129}130131export const now = () => performance.now();132133export function errInfo(e: any) {134  return {135    error: String(e?.message ?? e).slice(0, 600),136    errorType: e?.constructor?.name ?? typeof e,137    status: typeof e?.status === "number" ? e.status : undefined,138  };139}140141export function writeJSON(file: string, data: unknown) {142  fs.mkdirSync(path.dirname(file), { recursive: true });143  fs.writeFileSync(file, JSON.stringify(data, null, 2));144}145export function appendJSONL(file: string, data: unknown) {146  fs.mkdirSync(path.dirname(file), { recursive: true });147  fs.appendFileSync(file, JSON.stringify(data) + "\n");148}149150/** 64x64 solid red PNG for vision probes. */151export const RED_PNG_B64 =152  "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC";153154export async function pLimit<T>(n: number, tasks: (() => Promise<T>)[]): Promise<T[]> {155  const out: T[] = new Array(tasks.length);156  let i = 0;157  await Promise.all(Array.from({ length: Math.min(n, tasks.length) }, async () => {158    while (i < tasks.length) { const k = i++; out[k] = await tasks[k](); }159  }));160  return out;161}162