spb/llmindex Public

1/**2 * llmindex.io — CLI: pnpm eval:run --model <slug> --domain <domain> --n 200 [--k 1]3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { parseArgs } from 'node:util';8import { prisma } from '@llmindex/db';9import { isDomain } from '@llmindex/scoring';10import '../env';11import { runEvalBatch } from '../eval-runner';1213const { values } = parseArgs({14  options: {15    model: { type: 'string' },16    domain: { type: 'string' },17    n: { type: 'string', default: '200' },18    k: { type: 'string', default: '1' },19    seed: { type: 'string' },20  },21});2223async function main(): Promise<void> {24  if (!values.model || !values.domain) {25    console.error('Usage: pnpm eval:run --model <slug> --domain <domain> --n 200 [--k 1] [--seed s]');26    process.exit(2);27  }28  if (!isDomain(values.domain)) {29    console.error(`Unknown domain: ${values.domain}`);30    process.exit(2);31  }32  const result = await runEvalBatch({33    modelSlug: values.model,34    domain: values.domain,35    n: Number(values.n),36    kSamples: Number(values.k),37    seed: values.seed,38  });39  console.log(JSON.stringify(result));40}4142main()43  .catch((err) => {44    console.error(String(err));45    process.exitCode = 1;46  })47  .finally(() => prisma.$disconnect());48