spb/llmindex Public

1# llmindex.io — psychometrics tests: parameter recovery on synthetic data2# Author:  Simon-Pierre Boucher3# Contact: contact@spboucher.ai4# License: Proprietary — © Simon-Pierre Boucher, all rights reserved56import numpy as np7import pytest89from llmindex_psycho.bt import fit_bradley_terry10from llmindex_psycho.calibration import brier_score, expected_calibration_error11from llmindex_psycho.irt import fit_2pl121314def _synthetic_2pl(M=12, I=80, seed=7):15    rng = np.random.default_rng(seed)16    theta = rng.normal(0, 1, M)17    a = np.exp(rng.normal(0, 0.4, I))18    b = rng.normal(0, 1.2, I)19    P = 1 / (1 + np.exp(-a[None, :] * (theta[:, None] - b[None, :])))20    X = (rng.random((M, I)) < P).astype(float)21    return theta, a, b, X222324def test_2pl_recovers_ability_ordering():25    theta_true, _, _, X = _synthetic_2pl()26    result = fit_2pl(X, max_iterations=800)27    corr = np.corrcoef(theta_true, result.theta)[0, 1]28    assert corr > 0.85, f"theta recovery correlation too low: {corr:.3f}"29    assert np.all(result.theta_se > 0)30    assert np.all(result.a > 0)313233def test_2pl_handles_missing_cells():34    _, _, _, X = _synthetic_2pl()35    Xm = X.copy()36    rng = np.random.default_rng(1)37    Xm[rng.random(X.shape) < 0.3] = np.nan38    result = fit_2pl(Xm, max_iterations=600)39    assert np.isfinite(result.theta).all()40    assert np.isfinite(result.theta_se).all()414243def test_2pl_se_shrinks_with_more_items():44    _, _, _, X = _synthetic_2pl(M=8, I=120, seed=3)45    few = fit_2pl(X[:, :15], max_iterations=600)46    many = fit_2pl(X, max_iterations=600)47    assert many.theta_se.mean() < few.theta_se.mean()484950def test_2pl_rejects_empty():51    with pytest.raises(ValueError):52        fit_2pl(np.full((3, 3), np.nan))535455def test_bradley_terry_recovers_ordering():56    rng = np.random.default_rng(11)57    strength = np.array([2.0, 1.0, 0.0, -1.0, -2.0])58    M = len(strength)59    wins = np.zeros((M, M))60    for i in range(M):61        for j in range(M):62            if i == j:63                continue64            p = 1 / (1 + np.exp(-(strength[i] - strength[j])))65            wins[i, j] = rng.binomial(40, p)66    result = fit_bradley_terry(wins)67    assert result.converged68    assert list(np.argsort(-result.log_strength)) == [0, 1, 2, 3, 4]697071def test_calibration_metrics():72    conf = np.array([0.9, 0.9, 0.1, 0.1])73    correct = np.array([1.0, 1.0, 0.0, 0.0])74    assert brier_score(conf, correct) == pytest.approx(0.01)75    assert expected_calibration_error(conf, correct) == pytest.approx(0.1)76    overconfident = np.full(100, 0.99)77    outcomes = np.zeros(100)78    assert expected_calibration_error(overconfident, outcomes) > 0.979