spb/llmindex Public

1/**2 * llmindex.io — eval batch runner: perturbed items → OpenRouter → graded responses3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { createHash } from 'node:crypto';8import { Resvg } from '@resvg/resvg-js';9import { prisma } from '@llmindex/db';10import { extractAnswer, extractBlockAnswer, generateBatch, gradeAnswer } from '@llmindex/items';11import { OpenRouterClient, type ChatMessage, type Pricing } from '@llmindex/openrouter';12import { INDEX_VERSION, type Domain } from '@llmindex/scoring';13import { maxRunCostUsd } from './env';1415const CONCURRENCY = 4;16/** Batch is flagged degraded when more than 2% of calls fail (§6). */17export const DEGRADED_FAILURE_RATE = 0.02;18/** Conservative per-call token estimate for the pre-flight cost gate. */19export const EST_TOKENS = { prompt: 1200, completion: 1500 };20/** Completion budget: reasoning models need headroom or answers truncate (measured in the eval-harness literature). */21export const MAX_COMPLETION_TOKENS = 16384;2223export interface EvalBatchOptions {24  modelSlug: string;25  domain: Domain;26  n: number;27  /** k>1 adds consistency samples at the model's default temperature. */28  kSamples?: number;29  seed?: string;30  /** Live progress callback (completed calls, total calls) — best-effort. */31  onProgress?: (done: number, total: number) => void | Promise<void>;32}3334export function estimateBatchCostUsd(35  n: number,36  kSamples: number,37  pricing: Pricing,38): number {39  const calls = n + n * Math.max(0, kSamples - 1);40  return (41    (calls * (EST_TOKENS.prompt * pricing.promptPerM + EST_TOKENS.completion * pricing.completionPerM)) /42    1_000_00043  );44}4546export class CostCapError extends Error {}4748async function mapLimit<T, R>(49  items: T[],50  limit: number,51  fn: (item: T, index: number) => Promise<R>,52): Promise<R[]> {53  const results: R[] = new Array(items.length);54  let next = 0;55  const lanes = Array.from({ length: Math.min(limit, items.length) }, async () => {56    for (;;) {57      const i = next++;58      if (i >= items.length) return;59      results[i] = await fn(items[i]!, i);60    }61  });62  await Promise.all(lanes);63  return results;64}6566export async function runEvalBatch(opts: EvalBatchOptions): Promise<{ runId: string; status: string }> {67  const kSamples = opts.kSamples ?? 1;68  const model = await prisma.model.findUnique({ where: { slug: opts.modelSlug } });69  if (!model) throw new Error(`Model not in DB (run db:seed to sync): ${opts.modelSlug}`);70  if (model.promptPricePerM == null || model.completionPricePerM == null) {71    throw new Error(`Model has no pricing (cost tracking mandatory): ${opts.modelSlug}`);72  }73  const pricing: Pricing = {74    promptPerM: model.promptPricePerM,75    completionPerM: model.completionPricePerM,76  };7778  const estimated = estimateBatchCostUsd(opts.n, kSamples, pricing);79  const cap = maxRunCostUsd();80  if (estimated > cap) {81    throw new CostCapError(82      `Estimated batch cost $${estimated.toFixed(2)} exceeds MAX_RUN_COST_USD=$${cap} — refusing to start`,83    );84  }8586  const seed = opts.seed ?? `batch:${Date.now()}`;87  const items = generateBatch({ domain: opts.domain, n: opts.n, seed });88  const itemSetHash = createHash('sha256')89    .update(items.map((i) => i.perturbSeed).join('|'))90    .digest('hex');9192  const run = await prisma.scoreRun.create({93    data: {94      indexVersion: INDEX_VERSION,95      kind: 'eval_batch',96      status: 'running',97      itemSetHash,98      modelSet: [model.slug],99      notes: `domain=${opts.domain} n=${opts.n} k=${kSamples} seed=${seed}`,100    },101  });102103  const dbItems = await Promise.all(104    items.map((item) =>105      prisma.evalItem.create({106        data: {107          templateId: item.templateId,108          domain: item.domain,109          prompt: item.prompt,110          answerKey: item.answerKey,111          perturbSeed: item.perturbSeed,112          isAnchor: item.isAnchor,113        },114      }),115    ),116  );117118  const client = new OpenRouterClient();119  let failures = 0;120  let completedCalls = 0;121  // Small-context models reject max_tokens near/above their window (HTTP 400):122  // cap the completion budget by the context length, keeping prompt headroom.123  const maxTokens = model.contextLength124    ? Math.min(MAX_COMPLETION_TOKENS, Math.max(2048, model.contextLength - 4096))125    : MAX_COMPLETION_TOKENS;126127  interface Call {128    itemIdx: number;129    sampleIndex: number;130  }131  const calls: Call[] = [];132  for (let i = 0; i < items.length; i++) {133    for (let s = 0; s < kSamples; s++) calls.push({ itemIdx: i, sampleIndex: s });134  }135136  await mapLimit(calls, CONCURRENCY, async ({ itemIdx, sampleIndex }) => {137    const item = items[itemIdx]!;138    const dbItem = dbItems[itemIdx]!;139    // Vision items: rasterize the generated SVG scene → PNG data URL.140    let messages: ChatMessage[];141    if (item.svg) {142      const png = new Resvg(item.svg, { fitTo: { mode: 'width', value: 1024 } }).render().asPng();143      messages = [144        {145          role: 'user',146          content: [147            { type: 'text', text: item.prompt },148            { type: 'image_url', image_url: { url: `data:image/png;base64,${png.toString('base64')}` } },149          ],150        },151      ];152    } else {153      messages = [{ role: 'user', content: item.prompt }];154    }155    // Scored sample: temperature 0. Consistency samples: model default temperature (§6).156    const requestParams =157      sampleIndex === 0158        ? { model: model.slug, messages, temperature: 0, max_tokens: maxTokens }159        : { model: model.slug, messages, max_tokens: maxTokens };160    // Audit copy: keep exact params but elide the base64 image payload (the161    // scene is reproducible from the item's stored SVG/perturb seed).162    const auditParams = item.svg163      ? {164          ...requestParams,165          messages: [{ role: 'user', content: [{ type: 'text', text: item.prompt }, { type: 'image_url', image_url: { url: '[generated png elided — reproducible from eval_item svg]' } }] }],166        }167      : requestParams;168    try {169      const result = await client.chat(requestParams, pricing);170      const blockMode = item.grading === 'json' || item.grading === 'lines';171      const { answer, confidence } = blockMode172        ? extractBlockAnswer(result.text)173        : extractAnswer(result.text);174      // Truncated completions are unscored (correct=null), never "wrong":175      // grading a cut-off chain of thought measures the budget, not the model.176      const truncated = result.raw.choices?.[0]?.finish_reason === 'length' && answer === null;177      const correct = truncated ? null : gradeAnswer(answer, item.answerKey, item.grading);178      await prisma.modelResponse.create({179        data: {180          runId: run.id,181          modelId: model.id,182          itemId: dbItem.id,183          sampleIndex,184          requestParams: auditParams as object,185          rawResponse: result.raw as unknown as object,186          answerExtracted: answer,187          correct,188          error: truncated ? 'truncated' : null,189          confidence,190          tokensIn: result.usage?.prompt_tokens ?? null,191          tokensOut: result.usage?.completion_tokens ?? null,192          latencyMs: result.latencyMs,193          costUsd: result.costUsd,194        },195      });196    } catch (err) {197      failures += 1;198      await prisma.modelResponse.create({199        data: {200          runId: run.id,201          modelId: model.id,202          itemId: dbItem.id,203          sampleIndex,204          requestParams: auditParams as object,205          rawResponse: {},206          error: String(err).slice(0, 2000),207        },208      });209    }210    completedCalls += 1;211    if (opts.onProgress && (completedCalls % 5 === 0 || completedCalls === calls.length)) {212      try {213        await opts.onProgress(completedCalls, calls.length);214      } catch {215        /* progress is best-effort */216      }217    }218  });219220  const failureRate = failures / calls.length;221  const status = failureRate > DEGRADED_FAILURE_RATE ? 'degraded' : 'complete';222  await prisma.scoreRun.update({223    where: { id: run.id },224    data: {225      status,226      completedAt: new Date(),227      fitDiagnostics: { calls: calls.length, failures, failureRate },228    },229  });230  console.log(231    `[eval] run ${run.id} ${status}: model=${model.slug} domain=${opts.domain} calls=${calls.length} failures=${failures}`,232  );233  return { runId: run.id, status };234}235