spb/llmindex Public

1/**2 * llmindex.io — answer/confidence extraction and grading (robust cascade)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Extraction must never confound formatting with ability: models wrap answers8 * in markdown bold, backticks, LaTeX \boxed{}, or drop the tag entirely.9 * A lenient, ordered cascade extracts the intended answer; grading then10 * normalizes aggressively. Format compliance is measured by its own domain11 * (instruction_following), not silently through every other domain.12 */1314/** Instruction appended to every graded item prompt. */15export const ANSWER_FORMAT_INSTRUCTIONS =16  'End your reply with exactly two plain-text lines (no markdown, no extra text after them):\n' +17  'ANSWER: <your final answer only>\n' +18  'CONFIDENCE: <integer 0-100, how confident you are that your answer is correct>';1920/** Instruction for items whose answer is a JSON payload or multi-line output. */21export const BLOCK_ANSWER_FORMAT_INSTRUCTIONS =22  'Give your final answer inside ONE fenced code block (```), containing exactly the required ' +23  'content and nothing else. After the code block, add one plain-text line:\n' +24  'CONFIDENCE: <integer 0-100, how confident you are that your answer is correct>';2526export interface ExtractedAnswer {27  answer: string | null;28  /** Confidence in [0,1], null if the model did not report one. */29  confidence: number | null;30}3132/** Strip markdown decorations that models wrap around tags and values. */33function stripMd(s: string): string {34  return s35    .replace(/\*\*|__|~~|`+/g, '')36    .replace(/^\s*[#>*-]+\s*/, '')37    .trim();38}3940/** Unwrap LaTeX decorations: \boxed{x}, \text{x}, $x$, \( x \). */41function stripLatex(s: string): string {42  let out = s.trim();43  const boxed = out.match(/\\boxed\s*\{([^{}]*)\}/);44  if (boxed?.[1] != null) out = boxed[1];45  out = out46    .replace(/\\text\s*\{([^{}]*)\}/g, '$1')47    .replace(/\\mathrm\s*\{([^{}]*)\}/g, '$1')48    .replace(/^\$+|\$+$/g, '')49    .replace(/^\\\(|\\\)$/g, '')50    .trim();51  return out;52}5354function parseConfidence(text: string): number | null {55  const lines = text.split('\n').map(stripMd);56  const matches = lines57    .map((l) => l.match(/^CONFIDENCE\s*[:=]?\s*(\d{1,3})\s*%?\s*\.?$/i))58    .filter((m): m is RegExpMatchArray => m !== null);59  const last = matches[matches.length - 1];60  if (!last) return null;61  const v = Number(last[1]);62  return Number.isFinite(v) && v >= 0 && v <= 100 ? v / 100 : null;63}6465/**66 * Cascade for line answers:67 *  1. last markdown-stripped line matching "ANSWER: x" (also FINAL ANSWER / Answer =)68 *  2. \boxed{x} anywhere (last occurrence)69 *  3. null — graders may apply their own last-resort fallback (numeric only)70 */71export function extractAnswer(text: string): ExtractedAnswer {72  const confidence = parseConfidence(text);73  const lines = text.split('\n').map(stripMd);74  const tagMatches = lines75    .map((l) => l.match(/^(?:FINAL\s+)?ANSWER\s*[:=]\s*(.+?)\s*$/i))76    .filter((m): m is RegExpMatchArray => m !== null);77  const lastTag = tagMatches[tagMatches.length - 1];78  if (lastTag?.[1]) {79    return { answer: stripLatex(lastTag[1]), confidence };80  }81  const boxed = [...text.matchAll(/\\boxed\s*\{([^{}]*)\}/g)];82  const lastBoxed = boxed[boxed.length - 1];83  if (lastBoxed?.[1]) {84    return { answer: lastBoxed[1].trim(), confidence };85  }86  return { answer: null, confidence };87}8889/**90 * Extraction for block answers (JSON call sequences, multi-line terminal91 * output): last fenced code block; falls back to text after the last92 * "ANSWER:" tag when the model skipped the fence.93 */94export function extractBlockAnswer(text: string): ExtractedAnswer {95  const confidence = parseConfidence(text);96  const fences = [...text.matchAll(/```[a-zA-Z]*\r?\n([\s\S]*?)```/g)];97  const last = fences[fences.length - 1];98  if (last?.[1] != null && last[1].trim().length > 0) {99    return { answer: last[1].replace(/\s+$/, ''), confidence };100  }101  const tag = text.match(/(?:FINAL\s+)?ANSWER\s*[:=]\s*([\s\S]+)$/i);102  if (tag?.[1]) {103    return { answer: tag[1].replace(/CONFIDENCE\s*[:=][\s\S]*$/i, '').trim(), confidence };104  }105  return { answer: null, confidence };106}107108export function normalizeAnswer(raw: string): string {109  return stripLatex(stripMd(raw))110    .toLowerCase()111    .replace(/[""''`´]/g, '')112    .replace(/[\s.,;:!?]+$/g, '')113    .replace(/-/g, ' ') // hyphen/space orthography variants (vingt-deux ≡ vingt deux)114    .replace(/\s+/g, ' ')115    .trim();116}117118function parseNumeric(raw: string): number | null {119  let cleaned = stripLatex(stripMd(raw))120    .replace(/[$€£%]/g, '')121    .replace(/[\u00a0\u202f]/g, ' ')122    .trim();123  // thousands separators: "1,234,567" or "1 234 567"124  cleaned = cleaned.replace(/(\d)[ ,](?=\d{3}(\D|$))/g, '$1');125  cleaned = cleaned.replace(/[a-zA-Z]+$/g, '').trim(); // trailing units126  const m = cleaned.match(/-?\d+(?:\.\d+)?/);127  if (!m) return null;128  if (cleaned !== m[0]) {129    // reject if extra numeric garbage surrounds ("12 or 13")130    const rest = cleaned.replace(m[0], '');131    if (/\d/.test(rest)) return null;132  }133  const v = Number(m[0]);134  return Number.isFinite(v) ? v : null;135}136137/** Canonicalize JSON: stable key order, numbers as-is, whitespace-free. */138export function canonicalJson(value: unknown): string {139  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;140  if (value !== null && typeof value === 'object') {141    const entries = Object.entries(value as Record<string, unknown>)142      .filter(([, v]) => v !== undefined)143      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));144    return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`;145  }146  return JSON.stringify(value);147}148149function tryParseJson(raw: string): unknown | undefined {150  const cleaned = raw151    .replace(/```[a-zA-Z]*\r?\n?/g, '')152    .replace(/```/g, '')153    .trim();154  const start = cleaned.search(/[[{]/);155  if (start === -1) return undefined;156  const candidate = cleaned.slice(start);157  try {158    return JSON.parse(candidate);159  } catch {160    // trim trailing prose after the JSON payload161    for (let end = candidate.length; end > 1; end--) {162      const c = candidate[end - 1];163      if (c === ']' || c === '}') {164        try {165          return JSON.parse(candidate.slice(0, end));166        } catch {167          /* keep scanning */168        }169      }170    }171    return undefined;172  }173}174175export type GradingMode = 'exact' | 'numeric' | 'json' | 'lines' | 'constraints';176177/**178 * Constraint-stack spec (instruction_following): the answer key is a JSON179 * spec; ANY output satisfying every constraint is correct. All checks are180 * mechanical — no judges.181 */182export interface ConstraintSpec {183  wordCount?: number;184  startsWithWord?: string;185  endsWithWord?: string;186  includeWordExactly?: Array<{ word: string; count: number }>;187  forbiddenLetter?: string;188  allLowercase?: boolean;189}190191export function checkConstraints(output: string, spec: ConstraintSpec): boolean {192  const text = output.trim();193  const words = text194    .toLowerCase()195    .replace(/[.,;:!?"()]/g, ' ')196    .split(/\s+/)197    .filter(Boolean);198  if (spec.wordCount !== undefined && words.length !== spec.wordCount) return false;199  if (spec.startsWithWord && words[0] !== spec.startsWithWord.toLowerCase()) return false;200  if (spec.endsWithWord && words[words.length - 1] !== spec.endsWithWord.toLowerCase()) return false;201  if (spec.includeWordExactly) {202    for (const { word, count } of spec.includeWordExactly) {203      if (words.filter((w) => w === word.toLowerCase()).length !== count) return false;204    }205  }206  if (spec.forbiddenLetter && text.toLowerCase().includes(spec.forbiddenLetter.toLowerCase()))207    return false;208  if (spec.allLowercase && text !== text.toLowerCase()) return false;209  return true;210}211212/** Grade an extracted answer against a key. */213export function gradeAnswer(extracted: string | null, answerKey: string, grading: GradingMode): boolean {214  if (extracted === null) return false;215  switch (grading) {216    case 'numeric': {217      const got = parseNumeric(extracted);218      const want = parseNumeric(answerKey);219      if (got === null || want === null) return false;220      return Math.abs(got - want) <= Math.max(1e-9, Math.abs(want) * 1e-6);221    }222    case 'json': {223      const got = tryParseJson(extracted);224      const want = tryParseJson(answerKey);225      if (got === undefined || want === undefined) return false;226      return canonicalJson(got) === canonicalJson(want);227    }228    case 'constraints': {229      try {230        return checkConstraints(extracted, JSON.parse(answerKey) as ConstraintSpec);231      } catch {232        return false;233      }234    }235    case 'lines': {236      const clean = (s: string): string =>237        s238          .split('\n')239          .map((l) => l.replace(/\s+$/g, ''))240          .filter((l, i, arr) => !(l === '' && (i === 0 || i === arr.length - 1)))241          .join('\n')242          .trim();243      return clean(extracted) === clean(answerKey);244    }245    default:246      return normalizeAnswer(extracted) === normalizeAnswer(answerKey);247  }248}249