/** * llmindex.io — judge configuration for pairwise duels * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * Judge model slugs are configured via the JUDGE_MODELS env var (comma- * separated OpenRouter slugs) — never hardcoded in source (§6). Protocol * requirements enforced here (§6/§7): * - at least 2 judges from DIFFERENT providers * - position swapping on every duel * - a model never judges duels involving itself */ export interface JudgeConfig { slugs: string[]; } export class JudgeConfigError extends Error {} export function loadJudgeConfig(env: NodeJS.ProcessEnv = process.env): JudgeConfig { const raw = (env.JUDGE_MODELS ?? '').split(',').map((s) => s.trim()).filter(Boolean); if (raw.length < 2) { throw new JudgeConfigError( 'JUDGE_MODELS must list at least 2 OpenRouter slugs (comma-separated) from different providers', ); } const providers = new Set(raw.map((s) => s.split('/')[0])); if (providers.size < 2) { throw new JudgeConfigError('JUDGE_MODELS must span at least 2 different providers'); } return { slugs: raw }; } /** Judges eligible for a duel: never a participant. */ export function eligibleJudges(config: JudgeConfig, modelA: string, modelB: string): string[] { return config.slugs.filter((j) => j !== modelA && j !== modelB); } /** Rubric-free comparison prompt (public: docs/methodology/JUDGE_PROTOCOL.md). */ export function judgePrompt(task: string, responseFirst: string, responseSecond: string): string { return [ 'You are judging two anonymous assistant responses to the same task.', 'Judge holistically: task fulfillment, quality, precision, and appropriate handling.', 'Do not reward verbosity. Do not reward style over substance.', '', `TASK:\n${task}`, '', `RESPONSE 1:\n${responseFirst}`, '', `RESPONSE 2:\n${responseSecond}`, '', 'Reply with exactly one line: VERDICT: 1, VERDICT: 2, or VERDICT: TIE', ].join('\n'); } export function parseVerdict(text: string): '1' | '2' | 'tie' | null { const m = text.match(/VERDICT:\s*(1|2|TIE)/i); if (!m) return null; const v = m[1]!.toUpperCase(); return v === 'TIE' ? 'tie' : (v as '1' | '2'); }