/** * llmindex.io — index version, weights and IRT hyperparameters (single source of truth) * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * NEVER hardcode weights elsewhere. Every change here bumps INDEX_VERSION * (semver) and adds an entry to docs/methodology/CHANGELOG.md. */ import { DOMAINS, type Domain } from './domains'; export const INDEX_VERSION = '0.2.0'; /** 2PL IRT fit hyperparameters (consumed by apps/psychometrics via /api/v1/methodology). */ export const IRT_HYPERPARAMS = { model: '2PL', priors: { theta: { dist: 'normal', mean: 0, sd: 1 }, difficulty_b: { dist: 'normal', mean: 0, sd: 1.5 }, log_discrimination_a: { dist: 'normal', mean: 0, sd: 0.5 }, }, maxIterations: 500, tolerance: 1e-6, /** Items below this discrimination are auto-flagged for retirement review. */ minDiscrimination: 0.3, /** Items with |b| beyond this many logits are auto-flagged. */ maxAbsDifficultyLogits: 3, /** Fixed anchor subset is capped at this fraction of any scored run. */ maxAnchorFraction: 0.2, } as const; /** * Sub-metric weights inside a domain composite. Latency and cost are NEVER * blended in — they live on the efficiency frontier (Pareto), by design. * * Rationale (v0.2.0): accuracy_irt is the latent-ability estimate — the * primary construct — and dominates (0.60). Consistency (answer flip-rate * across seeded re-instantiations) and contamination resistance are * robustness corrections grounded in the template-memorization literature * (0.15 each); calibration rewards honest uncertainty but is the noisiest * sub-measurement at current sample sizes (0.10). Domain weights stay EQUAL: * with no task-utility function, the maximum-entropy prior is the only * non-arbitrary choice — per-domain scores are always published so any * consumer can re-weight. */ export const SUBMETRIC_WEIGHTS = { accuracy_irt: 0.6, consistency: 0.15, calibration: 0.1, contamination_resistance: 0.15, } as const; export type SubMetricKey = keyof typeof SUBMETRIC_WEIGHTS; /** Domain weights for the Global Index (equal in v1). */ export const DOMAIN_WEIGHTS: Record = Object.fromEntries( DOMAINS.map((d) => [d, 1 / DOMAINS.length]), ) as Record; /** * contamination_delta (fixed-vs-perturbed accuracy gap, in [0,1]) maps to a * resistance score: resistance = clamp01(1 - delta / CONTAMINATION_DELTA_FLOOR). * A gap ≥ 20 accuracy points ⇒ resistance 0. */ export const CONTAMINATION_DELTA_FLOOR = 0.2; /** θ → 0–1000 rescale for display: INDEX = CENTER + SLOPE·θ, clamped. */ export const THETA_SCALE = { center: 500, slope: 150, min: 0, max: 1000 } as const; /** CI half-width multiplier (95% normal). */ export const CI_Z = 1.96;