spb/llmindex Public

1/**2 * llmindex.io — judge configuration for pairwise duels3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Judge model slugs are configured via the JUDGE_MODELS env var (comma-8 * separated OpenRouter slugs) — never hardcoded in source (§6). Protocol9 * requirements enforced here (§6/§7):10 *   - at least 2 judges from DIFFERENT providers11 *   - position swapping on every duel12 *   - a model never judges duels involving itself13 */1415export interface JudgeConfig {16  slugs: string[];17}1819export class JudgeConfigError extends Error {}2021export function loadJudgeConfig(env: NodeJS.ProcessEnv = process.env): JudgeConfig {22  const raw = (env.JUDGE_MODELS ?? '').split(',').map((s) => s.trim()).filter(Boolean);23  if (raw.length < 2) {24    throw new JudgeConfigError(25      'JUDGE_MODELS must list at least 2 OpenRouter slugs (comma-separated) from different providers',26    );27  }28  const providers = new Set(raw.map((s) => s.split('/')[0]));29  if (providers.size < 2) {30    throw new JudgeConfigError('JUDGE_MODELS must span at least 2 different providers');31  }32  return { slugs: raw };33}3435/** Judges eligible for a duel: never a participant. */36export function eligibleJudges(config: JudgeConfig, modelA: string, modelB: string): string[] {37  return config.slugs.filter((j) => j !== modelA && j !== modelB);38}3940/** Rubric-free comparison prompt (public: docs/methodology/JUDGE_PROTOCOL.md). */41export function judgePrompt(task: string, responseFirst: string, responseSecond: string): string {42  return [43    'You are judging two anonymous assistant responses to the same task.',44    'Judge holistically: task fulfillment, quality, precision, and appropriate handling.',45    'Do not reward verbosity. Do not reward style over substance.',46    '',47    `TASK:\n${task}`,48    '',49    `RESPONSE 1:\n${responseFirst}`,50    '',51    `RESPONSE 2:\n${responseSecond}`,52    '',53    'Reply with exactly one line: VERDICT: 1, VERDICT: 2, or VERDICT: TIE',54  ].join('\n');55}5657export function parseVerdict(text: string): '1' | '2' | 'tie' | null {58  const m = text.match(/VERDICT:\s*(1|2|TIE)/i);59  if (!m) return null;60  const v = m[1]!.toUpperCase();61  return v === 'TIE' ? 'tie' : (v as '1' | '2');62}63