/** * llmindex.io — item bank entrypoint: template registry + batch generation * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved */ import type { Domain } from '@llmindex/scoring'; import { IRT_HYPERPARAMS } from '@llmindex/scoring'; import { createRng } from './rng'; import type { GeneratedItem, ItemTemplate } from './types'; import { mathArithChain, mathChained, mathCounterfactualBase, mathLinearSolve, mathPercentChain, } from './templates/math'; import { codeTraceJs, codeTraceNested, codeTracePython } from './templates/code'; import { reasoningOrder, reasoningSchedule } from './templates/reasoning'; import { knowledgeMc } from './templates/knowledge'; import { ifAcronym, ifConstraintStack, ifRepeat } from './templates/instruction'; import { multilingualNumword, multilingualWordToNum } from './templates/multilingual'; import { agenticDeploy, agenticLedger, agenticTriage } from './templates/agentic'; import { agenticContextLoad } from './templates/agentic-load'; import { terminalExitChain, terminalPipeline, terminalTree } from './templates/terminal'; import { visionCodeHunt, visionTableRead } from './templates/vision'; export * from './rng'; export * from './types'; export * from './answer'; export { generateSafetyDuelPrompt, generateSvgDuelPrompt, generateWritingDuelPrompt, } from './templates/duels'; export const TEMPLATES: readonly ItemTemplate[] = [ mathArithChain, mathLinearSolve, mathPercentChain, mathCounterfactualBase, mathChained, codeTracePython, codeTraceNested, codeTraceJs, reasoningOrder, reasoningSchedule, agenticTriage, agenticLedger, agenticDeploy, agenticContextLoad, terminalPipeline, terminalTree, terminalExitChain, visionCodeHunt, visionTableRead, knowledgeMc, ifRepeat, ifAcronym, ifConstraintStack, multilingualNumword, multilingualWordToNum, ]; export function templatesForDomain(domain: Domain): ItemTemplate[] { return TEMPLATES.filter((t) => t.domain === domain); } export interface GenerateBatchOptions { domain: Domain; n: number; /** Batch-level seed: same seed ⇒ identical batch (audit reproducibility). */ seed: string; /** * Fraction of items generated from the fixed anchor stream (longitudinal * comparability). Capped at IRT_HYPERPARAMS.maxAnchorFraction (≤20%). */ anchorFraction?: number; } export interface GeneratedBatchItem extends GeneratedItem { isAnchor: boolean; } /** * Generate a freshly-perturbed batch. Anchor items derive from a FIXED seed * stream (stable across runs); the rest derive from the batch seed (fresh * every run) — the accuracy gap between the two is contamination_delta. */ export function generateBatch(opts: GenerateBatchOptions): GeneratedBatchItem[] { const templates = templatesForDomain(opts.domain); if (templates.length === 0) { throw new Error(`No graded templates for domain "${opts.domain}" (duel-based domain?)`); } const anchorFraction = Math.min( opts.anchorFraction ?? 0.15, IRT_HYPERPARAMS.maxAnchorFraction, ); const nAnchor = Math.floor(opts.n * anchorFraction); const items: GeneratedBatchItem[] = []; for (let i = 0; i < opts.n; i++) { const isAnchor = i < nAnchor; const perturbSeed = isAnchor ? `anchor:${opts.domain}:${i}` // fixed stream — identical across runs : `${opts.seed}:${opts.domain}:${i}`; const rng = createRng(perturbSeed); const template = templates[i % templates.length]!; items.push({ ...template.render(rng, perturbSeed), isAnchor }); } return items; }