spb/llmindex Public

1# llmindex.io — calibration metrics: Brier score, expected calibration error2# Author:  Simon-Pierre Boucher3# Contact: contact@spboucher.ai4# License: Proprietary — © Simon-Pierre Boucher, all rights reserved56from __future__ import annotations78import numpy as np91011def brier_score(confidence: np.ndarray, correct: np.ndarray) -> float:12    """Mean squared error between reported confidence and outcome."""13    c = np.asarray(confidence, dtype=float)14    y = np.asarray(correct, dtype=float)15    if c.shape != y.shape or c.size == 0:16        raise ValueError("confidence and correct must be same-shape, non-empty")17    return float(np.mean((c - y) ** 2))181920def expected_calibration_error(21    confidence: np.ndarray, correct: np.ndarray, bins: int = 1022) -> float:23    """Standard ECE with equal-width confidence bins."""24    c = np.asarray(confidence, dtype=float)25    y = np.asarray(correct, dtype=float)26    if c.shape != y.shape or c.size == 0:27        raise ValueError("confidence and correct must be same-shape, non-empty")28    edges = np.linspace(0, 1, bins + 1)29    idx = np.clip(np.digitize(c, edges[1:-1]), 0, bins - 1)30    ece = 0.031    for b in range(bins):32        sel = idx == b33        n = sel.sum()34        if n == 0:35            continue36        ece += (n / c.size) * abs(y[sel].mean() - c[sel].mean())37    return float(ece)38