spb/llmindex Public

1/**2 * llmindex.io — item bank entrypoint: template registry + batch generation3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import type { Domain } from '@llmindex/scoring';8import { IRT_HYPERPARAMS } from '@llmindex/scoring';9import { createRng } from './rng';10import type { GeneratedItem, ItemTemplate } from './types';11import {12  mathArithChain,13  mathChained,14  mathCounterfactualBase,15  mathLinearSolve,16  mathPercentChain,17} from './templates/math';18import { codeTraceJs, codeTraceNested, codeTracePython } from './templates/code';19import { reasoningOrder, reasoningSchedule } from './templates/reasoning';20import { knowledgeMc } from './templates/knowledge';21import { ifAcronym, ifConstraintStack, ifRepeat } from './templates/instruction';22import { multilingualNumword, multilingualWordToNum } from './templates/multilingual';23import { agenticDeploy, agenticLedger, agenticTriage } from './templates/agentic';24import { agenticContextLoad } from './templates/agentic-load';25import { terminalExitChain, terminalPipeline, terminalTree } from './templates/terminal';26import { visionCodeHunt, visionTableRead } from './templates/vision';2728export * from './rng';29export * from './types';30export * from './answer';31export {32  generateSafetyDuelPrompt,33  generateSvgDuelPrompt,34  generateWritingDuelPrompt,35} from './templates/duels';3637export const TEMPLATES: readonly ItemTemplate[] = [38  mathArithChain,39  mathLinearSolve,40  mathPercentChain,41  mathCounterfactualBase,42  mathChained,43  codeTracePython,44  codeTraceNested,45  codeTraceJs,46  reasoningOrder,47  reasoningSchedule,48  agenticTriage,49  agenticLedger,50  agenticDeploy,51  agenticContextLoad,52  terminalPipeline,53  terminalTree,54  terminalExitChain,55  visionCodeHunt,56  visionTableRead,57  knowledgeMc,58  ifRepeat,59  ifAcronym,60  ifConstraintStack,61  multilingualNumword,62  multilingualWordToNum,63];6465export function templatesForDomain(domain: Domain): ItemTemplate[] {66  return TEMPLATES.filter((t) => t.domain === domain);67}6869export interface GenerateBatchOptions {70  domain: Domain;71  n: number;72  /** Batch-level seed: same seed ⇒ identical batch (audit reproducibility). */73  seed: string;74  /**75   * Fraction of items generated from the fixed anchor stream (longitudinal76   * comparability). Capped at IRT_HYPERPARAMS.maxAnchorFraction (≤20%).77   */78  anchorFraction?: number;79}8081export interface GeneratedBatchItem extends GeneratedItem {82  isAnchor: boolean;83}8485/**86 * Generate a freshly-perturbed batch. Anchor items derive from a FIXED seed87 * stream (stable across runs); the rest derive from the batch seed (fresh88 * every run) — the accuracy gap between the two is contamination_delta.89 */90export function generateBatch(opts: GenerateBatchOptions): GeneratedBatchItem[] {91  const templates = templatesForDomain(opts.domain);92  if (templates.length === 0) {93    throw new Error(`No graded templates for domain "${opts.domain}" (duel-based domain?)`);94  }95  const anchorFraction = Math.min(96    opts.anchorFraction ?? 0.15,97    IRT_HYPERPARAMS.maxAnchorFraction,98  );99  const nAnchor = Math.floor(opts.n * anchorFraction);100  const items: GeneratedBatchItem[] = [];101  for (let i = 0; i < opts.n; i++) {102    const isAnchor = i < nAnchor;103    const perturbSeed = isAnchor104      ? `anchor:${opts.domain}:${i}` // fixed stream — identical across runs105      : `${opts.seed}:${opts.domain}:${i}`;106    const rng = createRng(perturbSeed);107    const template = templates[i % templates.length]!;108    items.push({ ...template.render(rng, perturbSeed), isAnchor });109  }110  return items;111}112