spb/llmindex Public

1/**2 * llmindex.io — live benchmark orchestrator: one model at a time, refit after each3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Runs graded eval batches model-by-model across all IRT domains, publishing8 * live progress to Redis and triggering an IRT refit after every completed9 * model — so the public leaderboard grows gradually while the run is online.10 * (The first real fit appears once ≥2 models are complete, replacing the demo.)11 */12import { prisma } from '@llmindex/db';13import { DUEL_DOMAINS, IRT_DOMAINS, VISION_DOMAINS, type Domain } from '@llmindex/scoring';14import { runEvalBatch } from './eval-runner';15import { runDuelBatch } from './duel-runner';16import { runRefit } from './refit';17import { writeProgress, type BenchmarkProgress } from './progress';1819export interface BenchmarkOptions {20  /** Items per domain per model. */21  n?: number;22  /** Samples per item (consistency). */23  k?: number;24  /** Explicit slugs; defaults to the ranked subset. */25  models?: string[];26  /**27   * Models evaluated concurrently (lanes). 4 lanes × 4 in-batch calls =28   * ≤16 concurrent OpenRouter requests — well inside paid-tier limits, and29   * every call retries 429s with exponential backoff regardless.30   */31  parallel?: number;32}3334export async function runBenchmark(opts: BenchmarkOptions = {}): Promise<void> {35  const n = opts.n ?? 30;36  const k = opts.k ?? 2;37  const ranked = opts.models?.length38    ? await prisma.model.findMany({ where: { slug: { in: opts.models } }, orderBy: { slug: 'asc' } })39    : await prisma.model.findMany({ where: { ranked: true }, orderBy: { slug: 'asc' } });40  if (ranked.length === 0) throw new Error('No models to benchmark (run db:seed first)');4142  const slugs = ranked.map((m) => m.slug);43  const seedBase = `bench:${Date.now()}`;44  const state: BenchmarkProgress = {45    active: true,46    started_at: new Date().toISOString(),47    models: slugs,48    completed: [],49    failed: [],50    current: null,51    last_refit_run_id: null,52  };53  await writeProgress(state);54  console.log(`[bench] starting: ${slugs.length} models × ${IRT_DOMAINS.length} domains (n=${n}, k=${k})`);5556  const visionBySlug = new Map(ranked.map((m) => [m.slug, m.vision]));57  const lanes = Math.max(1, Math.min(12, opts.parallel ?? 6));5859  // Global (model × domain) task pool, model-major so models finish (and hit60  // the leaderboard) roughly one at a time even with parallel lanes.61  interface Task {62    slug: string;63    domain: Domain;64    di: number;65    domainsTotal: number;66  }67  const tasks: Task[] = [];68  const remainingByModel = new Map<string, number>();69  const modelHadError = new Set<string>();70  for (const slug of slugs) {71    const domains: readonly Domain[] = visionBySlug.get(slug)72      ? IRT_DOMAINS73      : IRT_DOMAINS.filter((d) => !VISION_DOMAINS.includes(d));74    remainingByModel.set(slug, domains.length);75    domains.forEach((domain, di) =>76      tasks.push({ slug, domain, di: di + 1, domainsTotal: domains.length }),77    );78  }7980  const laneCurrent: (typeof state.current)[] = Array.from({ length: lanes }, () => null);81  let refitChain: Promise<void> = Promise.resolve();82  const syncProgress = async (): Promise<void> => {83    const currents = laneCurrent.filter((c): c is NonNullable<typeof c> => c !== null);84    state.currents = currents;85    state.current = currents[0] ?? null;86    await writeProgress(state);87  };8889  let cursor = 0;90  console.log(`[bench] parallel lanes=${lanes} → ≤${lanes * 4} concurrent OpenRouter calls`);91  await Promise.all(92    Array.from({ length: lanes }, (_, laneId) =>93      (async () => {94        for (;;) {95          const task = tasks[cursor++];96          if (!task) return;97          laneCurrent[laneId] = {98            model: task.slug,99            domain: task.domain,100            domain_index: task.di,101            domains_total: task.domainsTotal,102            calls_done: 0,103            calls_total: n * k,104          };105          await syncProgress();106          try {107            const result = await runEvalBatch({108              modelSlug: task.slug,109              domain: task.domain,110              n,111              kSamples: k,112              seed: `${seedBase}:${task.slug}:${task.domain}`,113              onProgress: async (done, total) => {114                const cur = laneCurrent[laneId];115                if (cur && cur.model === task.slug && cur.domain === task.domain) {116                  cur.calls_done = done;117                  cur.calls_total = total;118                  await syncProgress();119                }120              },121            });122            if (result.status === 'degraded') {123              console.warn(`[bench] ${task.slug}/${task.domain} degraded (excluded from fits)`);124            }125          } catch (err) {126            console.warn(`[bench] ${task.slug}/${task.domain} failed: ${String(err).slice(0, 300)}`);127            modelHadError.add(task.slug);128          }129          laneCurrent[laneId] = null;130          const left = (remainingByModel.get(task.slug) ?? 1) - 1;131          remainingByModel.set(task.slug, left);132          if (left === 0) {133            (modelHadError.has(task.slug) ? state.failed : state.completed).push(task.slug);134            // Refits are serialized (never concurrent) via a promise chain.135            refitChain = refitChain.then(async () => {136              try {137                const refit = await runRefit();138                if (refit) state.last_refit_run_id = refit.runId;139              } catch (err) {140                console.warn(`[bench] refit after ${task.slug} failed: ${String(err).slice(0, 300)}`);141              }142            });143            console.log(144              `[bench] ${task.slug} done (${state.completed.length + state.failed.length}/${slugs.length})`,145            );146          }147          await syncProgress();148        }149      })(),150    ),151  );152  await refitChain;153154  // Judged duel domains (writing, safety, svg_design): pairwise + 3-judge155  // panel + Bradley-Terry. Requires JUDGE_MODELS in the environment.156  if (process.env.JUDGE_MODELS) {157    const pairsPerDomain = Math.min(400, Math.max(60, slugs.length * 6));158    for (const domain of DUEL_DOMAINS) {159      state.current = {160        model: `duels:${domain}`,161        domain,162        domain_index: 1,163        domains_total: 1,164        calls_done: 0,165        calls_total: pairsPerDomain,166      };167      await writeProgress(state);168      try {169        await runDuelBatch({170          domain: domain as 'writing' | 'safety_refusal_quality' | 'svg_design',171          pairs: pairsPerDomain,172          seed: `${seedBase}:duel:${domain}`,173          onProgress: async (done, total) => {174            if (state.current) {175              state.current.calls_done = done;176              state.current.calls_total = total;177              await writeProgress(state);178            }179          },180        });181      } catch (err) {182        console.warn(`[bench] duel domain ${domain} failed: ${String(err).slice(0, 300)}`);183      }184    }185    state.current = null;186    try {187      const refit = await runRefit();188      if (refit) state.last_refit_run_id = refit.runId;189    } catch (err) {190      console.warn(`[bench] final refit failed: ${String(err).slice(0, 300)}`);191    }192  } else {193    console.warn('[bench] JUDGE_MODELS not set — skipping duel domains (writing/safety/svg)');194  }195196  state.active = false;197  state.note = `benchmark complete: ${state.completed.length} ok, ${state.failed.length} failed`;198  await writeProgress(state);199  console.log(`[bench] ${state.note}`);200}201