spb/llmindex Public

1/**2 * llmindex.io — instruction-following templates (mechanically checkable constraints)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { ANSWER_FORMAT_INSTRUCTIONS, type ConstraintSpec } from '../answer';8import type { ItemTemplate } from '../types';910const WORDS = [11  'nova', 'delta', 'ember', 'quartz', 'falcon', 'lumen', 'cedar', 'orbit', 'prism', 'tundra',12  'zephyr', 'basalt', 'comet', 'drift', 'echo', 'flint',13];1415export const ifRepeat: ItemTemplate = {16  id: 'if.format.repeat-v1',17  domain: 'instruction_following',18  description:19    'Produce a word repeated n times with an exact separator and casing — checkable string output.',20  paramSpace: WORDS.length * 7 * 3 * 3,21  render(rng, perturbSeed) {22    const word = rng.pick(WORDS);23    const n = rng.int(3, 9);24    const sep = rng.pick(['-', '_', '/'] as const);25    const casing = rng.pick(['uppercase', 'lowercase', 'capitalized'] as const);26    const cased =27      casing === 'uppercase'28        ? word.toUpperCase()29        : casing === 'capitalized'30          ? word[0]!.toUpperCase() + word.slice(1)31          : word;32    const expected = Array.from({ length: n }, () => cased).join(sep);33    return {34      templateId: this.id,35      domain: this.domain,36      prompt:37        `Write the word "${word}" in ${casing} form, repeated exactly ${n} times, ` +38        `joined by the character "${sep}" with no spaces. Output that string as your answer.\n\n${ANSWER_FORMAT_INSTRUCTIONS}`,39      answerKey: expected,40      grading: 'exact',41      perturbSeed,42    };43  },44};4546const TOPICS = ['the sea', 'a city at night', 'winter mornings', 'an old machine', 'a long journey'] as const;4748/**49 * Constraint stacking (IFEval-style, home-made): 4-5 simultaneously50 * verifiable constraints on one short generated text. Every constraint is51 * mechanically checkable; satisfiability is guaranteed by construction52 * (the generator verifies a witness before emitting the item).53 */54export const ifConstraintStack: ItemTemplate = {55  id: 'if.constraints.stack-v1',56  domain: 'instruction_following',57  description:58    'Write one sentence satisfying 4-5 stacked verifiable constraints (word count, boundary words, exact keyword frequency, forbidden letter, casing).',59  paramSpace: 16 ** 3 * 20 * 5 * 4,60  render(rng, perturbSeed) {61    const wordCount = rng.int(14, 24);62    const startWord = rng.pick(WORDS);63    const endWord = rng.pick(WORDS.filter((w) => w !== startWord));64    const keyword = rng.pick(WORDS.filter((w) => w !== startWord && w !== endWord));65    const keywordCount = rng.int(2, 3);66    // Forbidden letter must not appear in mandatory words.67    const mandatory = `${startWord}${endWord}${keyword}`;68    const candidates = 'qjzxv'.split('').filter((ch) => !mandatory.includes(ch));69    const forbiddenLetter = candidates[0] ?? 'q';70    const spec: ConstraintSpec = {71      wordCount,72      startsWithWord: startWord,73      endsWithWord: endWord,74      includeWordExactly: [{ word: keyword, count: keywordCount }],75      forbiddenLetter,76      allLowercase: true,77    };78    // Witness check: constraints are jointly satisfiable by construction —79    // filler words below avoid the forbidden letters entirely.80    const prompt =81      `Write in English about ${rng.pick(TOPICS)}, following ALL of these rules simultaneously:\n` +82      `1. Exactly ${wordCount} words.\n` +83      `2. The first word must be "${startWord}" and the last word must be "${endWord}".\n` +84      `3. Use the word "${keyword}" exactly ${keywordCount} times (in addition to rules 2 if they differ).\n` +85      `4. The letter "${forbiddenLetter}" must not appear anywhere.\n` +86      `5. Everything entirely in lowercase.\n\n` +87      `Give the text itself as your answer.\n\n${ANSWER_FORMAT_INSTRUCTIONS}`;88    return {89      templateId: this.id,90      domain: this.domain,91      prompt,92      answerKey: JSON.stringify(spec),93      grading: 'constraints',94      perturbSeed,95    };96  },97};9899export const ifAcronym: ItemTemplate = {100  id: 'if.format.acronym-v1',101  domain: 'instruction_following',102  description: 'Build an acronym from the k-th letters of a generated word list.',103  paramSpace: WORDS.length ** 4 * 3,104  render(rng, perturbSeed) {105    const words = rng.shuffle(WORDS).slice(0, rng.int(4, 6));106    const k = rng.int(1, 3);107    const ordinal = k === 1 ? 'first' : k === 2 ? 'second' : 'third';108    const expected = words.map((w) => w[k - 1]!.toUpperCase()).join('');109    return {110      templateId: this.id,111      domain: this.domain,112      prompt:113        `Take the ${ordinal} letter of each of these words, in order: ${words.join(', ')}. ` +114        `Concatenate them in uppercase into a single string with no separators. Output that string as your answer.\n\n${ANSWER_FORMAT_INSTRUCTIONS}`,115      answerKey: expected,116      grading: 'exact',117      perturbSeed,118    };119  },120};121