spb/llmindex Public

1/**2 * llmindex.io — index version, weights and IRT hyperparameters (single source of truth)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * NEVER hardcode weights elsewhere. Every change here bumps INDEX_VERSION8 * (semver) and adds an entry to docs/methodology/CHANGELOG.md.9 */10import { DOMAINS, type Domain } from './domains';1112export const INDEX_VERSION = '0.2.0';1314/** 2PL IRT fit hyperparameters (consumed by apps/psychometrics via /api/v1/methodology). */15export const IRT_HYPERPARAMS = {16  model: '2PL',17  priors: {18    theta: { dist: 'normal', mean: 0, sd: 1 },19    difficulty_b: { dist: 'normal', mean: 0, sd: 1.5 },20    log_discrimination_a: { dist: 'normal', mean: 0, sd: 0.5 },21  },22  maxIterations: 500,23  tolerance: 1e-6,24  /** Items below this discrimination are auto-flagged for retirement review. */25  minDiscrimination: 0.3,26  /** Items with |b| beyond this many logits are auto-flagged. */27  maxAbsDifficultyLogits: 3,28  /** Fixed anchor subset is capped at this fraction of any scored run. */29  maxAnchorFraction: 0.2,30} as const;3132/**33 * Sub-metric weights inside a domain composite. Latency and cost are NEVER34 * blended in — they live on the efficiency frontier (Pareto), by design.35 *36 * Rationale (v0.2.0): accuracy_irt is the latent-ability estimate — the37 * primary construct — and dominates (0.60). Consistency (answer flip-rate38 * across seeded re-instantiations) and contamination resistance are39 * robustness corrections grounded in the template-memorization literature40 * (0.15 each); calibration rewards honest uncertainty but is the noisiest41 * sub-measurement at current sample sizes (0.10). Domain weights stay EQUAL:42 * with no task-utility function, the maximum-entropy prior is the only43 * non-arbitrary choice — per-domain scores are always published so any44 * consumer can re-weight.45 */46export const SUBMETRIC_WEIGHTS = {47  accuracy_irt: 0.6,48  consistency: 0.15,49  calibration: 0.1,50  contamination_resistance: 0.15,51} as const;5253export type SubMetricKey = keyof typeof SUBMETRIC_WEIGHTS;5455/** Domain weights for the Global Index (equal in v1). */56export const DOMAIN_WEIGHTS: Record<Domain, number> = Object.fromEntries(57  DOMAINS.map((d) => [d, 1 / DOMAINS.length]),58) as Record<Domain, number>;5960/**61 * contamination_delta (fixed-vs-perturbed accuracy gap, in [0,1]) maps to a62 * resistance score: resistance = clamp01(1 - delta / CONTAMINATION_DELTA_FLOOR).63 * A gap ≥ 20 accuracy points ⇒ resistance 0.64 */65export const CONTAMINATION_DELTA_FLOOR = 0.2;6667/** θ → 0–1000 rescale for display: INDEX = CENTER + SLOPE·θ, clamped. */68export const THETA_SCALE = { center: 500, slope: 150, min: 0, max: 1000 } as const;6970/** CI half-width multiplier (95% normal). */71export const CI_Z = 1.96;72