/** * llmindex.io — knowledge item templates (fact bank + MC option shuffling + paraphrase) * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * Knowledge facts cannot be synthesized, so contamination resistance comes from * paraphrase variants + shuffled distractors; these templates are the main * anchor-eligible family and feed the contamination_delta measurement. */ import { ANSWER_FORMAT_INSTRUCTIONS } from '../answer'; import type { ItemTemplate } from '../types'; interface Fact { q: string[]; answer: string; distractors: string[]; } const CAPITALS: Fact[] = [ { q: ['capital of Australia', 'Australian capital city'], answer: 'Canberra', distractors: ['Sydney', 'Melbourne', 'Perth', 'Brisbane'] }, { q: ['capital of Canada', 'Canadian capital city'], answer: 'Ottawa', distractors: ['Toronto', 'Montreal', 'Vancouver', 'Calgary'] }, { q: ['capital of Brazil', 'Brazilian capital city'], answer: 'Brasília', distractors: ['Rio de Janeiro', 'São Paulo', 'Salvador', 'Recife'] }, { q: ['capital of Turkey', 'Turkish capital city'], answer: 'Ankara', distractors: ['Istanbul', 'Izmir', 'Antalya', 'Bursa'] }, { q: ['capital of Switzerland', 'Swiss capital (de facto)'], answer: 'Bern', distractors: ['Zurich', 'Geneva', 'Basel', 'Lausanne'] }, { q: ['capital of Kazakhstan', 'Kazakh capital city'], answer: 'Astana', distractors: ['Almaty', 'Shymkent', 'Karaganda', 'Aktobe'] }, { q: ['capital of Myanmar', 'Burmese capital city'], answer: 'Naypyidaw', distractors: ['Yangon', 'Mandalay', 'Bago', 'Taunggyi'] }, { q: ['capital of Nigeria', 'Nigerian capital city'], answer: 'Abuja', distractors: ['Lagos', 'Kano', 'Ibadan', 'Port Harcourt'] }, ]; const ELEMENTS: Fact[] = [ { q: ['chemical element with symbol W', 'element whose symbol is W'], answer: 'Tungsten', distractors: ['Tin', 'Titanium', 'Tantalum', 'Terbium'] }, { q: ['chemical element with symbol Hg', 'element whose symbol is Hg'], answer: 'Mercury', distractors: ['Hydrogen', 'Hafnium', 'Holmium', 'Helium'] }, { q: ['chemical element with symbol K', 'element whose symbol is K'], answer: 'Potassium', distractors: ['Krypton', 'Calcium', 'Phosphorus', 'Carbon'] }, { q: ['chemical element with symbol Sn', 'element whose symbol is Sn'], answer: 'Tin', distractors: ['Silicon', 'Selenium', 'Sodium', 'Scandium'] }, { q: ['chemical element with symbol Sb', 'element whose symbol is Sb'], answer: 'Antimony', distractors: ['Strontium', 'Sulfur', 'Silver', 'Samarium'] }, { q: ['chemical element with symbol Pb', 'element whose symbol is Pb'], answer: 'Lead', distractors: ['Platinum', 'Palladium', 'Polonium', 'Phosphorus'] }, ]; const AUTHORS: Fact[] = [ { q: ['author of "One Hundred Years of Solitude"', 'writer of the novel "One Hundred Years of Solitude"'], answer: 'Gabriel García Márquez', distractors: ['Mario Vargas Llosa', 'Jorge Luis Borges', 'Julio Cortázar', 'Isabel Allende'] }, { q: ['author of "Things Fall Apart"', 'writer of the novel "Things Fall Apart"'], answer: 'Chinua Achebe', distractors: ['Wole Soyinka', 'Ngũgĩ wa Thiong\'o', 'Ben Okri', 'Chimamanda Ngozi Adichie'] }, { q: ['author of "The Master and Margarita"', 'writer of the novel "The Master and Margarita"'], answer: 'Mikhail Bulgakov', distractors: ['Fyodor Dostoevsky', 'Leo Tolstoy', 'Boris Pasternak', 'Anton Chekhov'] }, { q: ['author of "Snow Country"', 'writer of the novel "Snow Country"'], answer: 'Yasunari Kawabata', distractors: ['Yukio Mishima', 'Haruki Murakami', 'Kenzaburō Ōe', 'Natsume Sōseki'] }, ]; const BANK: Fact[] = [...CAPITALS, ...ELEMENTS, ...AUTHORS]; const STEMS = [ 'What is the {Q}?', 'Name the {Q}.', 'Identify the {Q}.', ]; /** * Free-response (no options): removes the multiple-choice guessing floor * (c≈0 in IRT terms), restoring full item information at P≈0.5. Distractor * lists are retained for future buggy-solver MC variants but unused here. */ export const knowledgeMc: ItemTemplate = { id: 'knowledge.fr.factbank-v2', domain: 'knowledge', description: 'Free-response facts from a curated bank with paraphrased stems — no options, no guessing floor.', paramSpace: BANK.length * STEMS.length * 2, render(rng, perturbSeed) { const fact = rng.pick(BANK); const stem = rng.pick(STEMS).replace('{Q}', rng.pick(fact.q)); return { templateId: this.id, domain: this.domain, prompt: `${stem}\n\nAnswer with the name only — no explanation.\n\n${ANSWER_FORMAT_INSTRUCTIONS}`, answerKey: fact.answer, grading: 'exact', perturbSeed, }; }, };