spb/llmindex Public

1/**2 * llmindex.io — terminal item templates: home-made shell simulation, deterministic ground truth3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Original terminal-competence benchmark: no shell ever executes. A closed,8 * unambiguous POSIX subset (byte-order C-locale sort, integer-only awk forms,9 * fixed-string grep, no locale/format-dependent constructs) is simulated in10 * TypeScript; the model predicts exact output / final file tree / execution11 * traces, graded by exact match. Seeded generation of file contents and12 * pipelines makes memorization worthless.13 */14import { BLOCK_ANSWER_FORMAT_INSTRUCTIONS } from '../answer';15import type { Rng } from '../rng';16import type { ItemTemplate } from '../types';1718/* ----------------------------- shared simulator ----------------------------- */1920/** Byte-wise (C-locale) string comparison — ASCII-only data by construction. */21const byteCmp = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);2223type Lines = string[];2425interface PipeStage {26  text: string;27  fn: (input: Lines) => Lines;28}2930/* ------------------------------------------------------------------- *31 * Shape A: pipeline output prediction over a generated CSV             *32 * ------------------------------------------------------------------- */3334const DEPTS = ['sales', 'eng', 'ops', 'hr', 'legal'] as const;35const FIRST = ['ana', 'bo', 'cy', 'dev', 'eli', 'fay', 'gus', 'hal', 'ivy', 'jon', 'kim', 'lou', 'max', 'ned', 'oli', 'pam'] as const;3637export const terminalPipeline: ItemTemplate = {38  id: 'terminal.pipeline.predict-v1',39  domain: 'terminal',40  description:41    'Predict the exact stdout of a 3-5 stage pipeline (grep/cut/sort/head/tail/awk subset) over a generated CSV; byte-order vs numeric sort traps included.',42  paramSpace: 16 ** 10 * 5 ** 10 * 4 ** 5,43  render(rng: Rng, perturbSeed: string) {44    // Generate CSV rows: name,dept,units,score — names unique, ASCII lowercase.45    const nRows = rng.int(9, 14);46    const names = rng.shuffle(FIRST).slice(0, nRows);47    const rows = names.map((name) => {48      const dept = rng.pick(DEPTS);49      const units = rng.int(3, 120); // 1-3 digits → numeric-vs-byte sort trap50      const score = rng.int(10, 99);51      return `${name},${dept},${units},${score}`;52    });5354    // Build a pipeline of 3-5 stages from a closed, unambiguous set.55    const targetDept = rng.pick([...new Set(rows.map((r) => r.split(',')[1]!))]);56    const stages: PipeStage[] = [];57    stages.push({58      text: `grep -F ',${targetDept},' people.csv`,59      fn: (input) => input.filter((l) => l.includes(`,${targetDept},`)),60    });6162    const variant = rng.int(0, 3);63    if (variant === 0) {64      // numeric sort on units, take top rows65      const n = rng.int(2, 3);66      stages.push({67        text: `sort -t, -k3,3n`,68        fn: (input) =>69          [...input].sort((a, b) => Number(a.split(',')[2]) - Number(b.split(',')[2]) || byteCmp(a, b)),70      });71      stages.push({ text: `tail -n ${n}`, fn: (input) => input.slice(-n) });72    } else if (variant === 1) {73      // byte-order sort on the whole line (the classic 100 < 9 trap), head74      const n = rng.int(2, 3);75      stages.push({76        text: `cut -d, -f1,3`,77        fn: (input) => input.map((l) => l.split(',').filter((_, i) => i === 0 || i === 2).join(',')),78      });79      stages.push({ text: `sort`, fn: (input) => [...input].sort(byteCmp) });80      stages.push({ text: `head -n ${n}`, fn: (input) => input.slice(0, n) });81    } else if (variant === 2) {82      // awk integer aggregation → single number83      stages.push({84        text: `awk -F, '{ s += $3 } END { print s }'`,85        fn: (input) => [String(input.reduce((s, l) => s + Number(l.split(',')[2]), 0))],86      });87    } else {88      // awk filter + count via grep -c style END counter89      const cutoff = rng.int(40, 80);90      stages.push({91        text: `awk -F, '$4 > ${cutoff} { n += 1 } END { print n }'`,92        fn: (input) => [String(input.filter((l) => Number(l.split(',')[3]) > cutoff).length)],93      });94    }9596    let out: Lines = rows;97    for (const s of stages) out = s.fn(out);98    const pipeline = stages.map((s) => s.text).join(' | ');99100    const prompt = [101      'A POSIX shell session (LC_ALL=C). The file `people.csv` contains exactly these lines (columns: name,dept,units,score):',102      '',103      '```',104      rows.join('\n'),105      '```',106      '',107      'What is the EXACT stdout of this command?',108      '',109      '```sh',110      pipeline,111      '```',112      '',113      'Notes: plain `sort` compares bytes (so "100" sorts before "9"); `sort -k3,3n` compares field 3 numerically.',114      '',115      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,116    ].join('\n');117118    return {119      templateId: this.id,120      domain: this.domain,121      prompt,122      answerKey: out.join('\n'),123      grading: 'lines' as const,124      perturbSeed,125    };126  },127};128129/* ------------------------------------------------------------------- *130 * Shape B: file-tree prediction after mkdir/mv/cp/rm/touch + cd chain  *131 * ------------------------------------------------------------------- */132133const DIR_NAMES = ['src', 'docs', 'build', 'assets', 'logs', 'conf'] as const;134const FILE_STEMS = ['main', 'util', 'index', 'setup', 'notes', 'report', 'draft', 'todo'] as const;135const EXTS = ['txt', 'md', 'log', 'cfg'] as const;136137interface VirtualFs {138  files: Set<string>; // absolute paths like /proj/src/main.txt139  dirs: Set<string>; // absolute dir paths140  cwd: string;141}142143export const terminalTree: ItemTemplate = {144  id: 'terminal.fs.tree-v1',145  domain: 'terminal',146  description:147    'Track a virtual file tree through a sequence of mkdir/touch/mv/cp/rm with relative paths and cd; output the sorted final file list.',148  paramSpace: 6 ** 3 * 8 ** 5 * 4 ** 5 * 6 ** 6,149  render(rng: Rng, perturbSeed: string) {150    const fs: VirtualFs = { files: new Set(), dirs: new Set(['/proj']), cwd: '/proj' };151    const dirs = rng.shuffle(DIR_NAMES).slice(0, 3);152    for (const d of dirs) fs.dirs.add(`/proj/${d}`);153    const stems = rng.shuffle(FILE_STEMS).slice(0, 5);154    for (const [i, stem] of stems.entries()) {155      const dir = i < 2 ? '/proj' : `/proj/${rng.pick(dirs)}`;156      fs.files.add(`${dir}/${stem}.${rng.pick(EXTS)}`);157    }158159    const initialListing = [...fs.files].sort(byteCmp);160    const script: string[] = [];161    const nOps = rng.int(6, 8);162    for (let i = 0; i < nOps; i++) {163      const op = rng.int(0, 4);164      const fileArr = [...fs.files];165      if (op === 0 && fileArr.length > 1) {166        // mv file → other dir (move) or rename in place167        const f = rng.pick(fileArr);168        if (rng.next() < 0.5) {169          const destDir = rng.pick([...fs.dirs]);170          const dest = `${destDir}/${f.split('/').pop()}`;171          script.push(`mv ${rel(fs.cwd, f)} ${rel(fs.cwd, destDir)}/`);172          fs.files.delete(f);173          fs.files.add(dest);174        } else {175          const newName = `${rng.pick(FILE_STEMS)}-${rng.int(1, 9)}.${rng.pick(EXTS)}`;176          const dest = `${f.split('/').slice(0, -1).join('/')}/${newName}`;177          script.push(`mv ${rel(fs.cwd, f)} ${rel(fs.cwd, dest)}`);178          fs.files.delete(f);179          fs.files.add(dest);180        }181      } else if (op === 1) {182        const destDir = rng.pick([...fs.dirs]);183        const f = rng.pick(fileArr);184        const dest = `${destDir}/${f.split('/').pop()}`;185        if (dest !== f) {186          script.push(`cp ${rel(fs.cwd, f)} ${rel(fs.cwd, destDir)}/`);187          fs.files.add(dest);188        }189      } else if (op === 2 && fileArr.length > 3) {190        const f = rng.pick(fileArr);191        script.push(`rm ${rel(fs.cwd, f)}`);192        fs.files.delete(f);193      } else if (op === 3) {194        const d = `${rng.pick([...fs.dirs])}/${rng.pick(DIR_NAMES)}-${rng.int(1, 9)}`;195        if (!fs.dirs.has(d)) {196          script.push(`mkdir -p ${rel(fs.cwd, d)}`);197          fs.dirs.add(d);198        }199      } else {200        const dir = rng.pick([...fs.dirs]);201        const f = `${dir}/${rng.pick(FILE_STEMS)}-${rng.int(1, 9)}.${rng.pick(EXTS)}`;202        if (!fs.files.has(f)) {203          script.push(`touch ${rel(fs.cwd, f)}`);204          fs.files.add(f);205        }206      }207      // occasionally change directory (forces relative-path tracking)208      if (rng.next() < 0.3) {209        const d = rng.pick([...fs.dirs]);210        script.push(`cd ${rel(fs.cwd, d) || '.'}`);211        fs.cwd = d;212      }213    }214215    function rel(cwd: string, abs: string): string {216      if (abs === cwd) return '.';217      if (abs.startsWith(cwd + '/')) return abs.slice(cwd.length + 1);218      // walk up from cwd to root then down — keep it simple and unambiguous219      const up = cwd.split('/').filter(Boolean).length;220      return '../'.repeat(up) + abs.replace(/^\//, '');221    }222223    const finalListing = [...fs.files].sort(byteCmp);224225    const prompt = [226      'A POSIX shell session starts in `/proj`. The tree initially contains these FILES (directories exist as implied, plus empty dirs ' +227        dirs.map((d) => `\`/proj/${d}\``).join(', ') +228        '):',229      '',230      '```',231      initialListing.join('\n'),232      '```',233      '',234      'These commands run in order (all succeed; `mv x dir/` moves into the directory; paths are relative to the CURRENT working directory, which `cd` changes):',235      '',236      '```sh',237      script.join('\n'),238      '```',239      '',240      'List every file (absolute paths) that exists afterwards, one per line, sorted in byte order (C locale). Do not list directories.',241      '',242      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,243    ].join('\n');244245    return {246      templateId: this.id,247      domain: this.domain,248      prompt,249      answerKey: finalListing.join('\n'),250      grading: 'lines' as const,251      perturbSeed,252    };253  },254};255256/* ------------------------------------------------------------------- *257 * Shape C: && / || short-circuit execution-trace prediction            *258 * ------------------------------------------------------------------- */259260export const terminalExitChain: ItemTemplate = {261  id: 'terminal.exit.chain-v1',262  domain: 'terminal',263  description:264    'Predict which echo statements run and the final exit status of a && / || chain over test -f / grep -q primitives with known outcomes.',265  paramSpace: 2 ** 8 * 8 ** 4 * 26 ** 2,266  render(rng: Rng, perturbSeed: string) {267    // Known world: files that exist + a haystack file with known content.268    const present = ['app.txt', 'data.txt'].filter(() => rng.next() < 0.8);269    const absent = ['ghost.txt', 'tmp.txt'];270    const words = ['amber', 'basil', 'coral', 'dune'];271    const inHaystack = rng.shuffle(words).slice(0, 2);272    const notInHaystack = words.filter((w) => !inHaystack.includes(w));273274    interface Prim {275      text: string;276      ok: boolean;277      echo?: string;278    }279    const prims: Prim[] = [];280    const nSegments = rng.int(3, 4);281    let letter = 65; // A, B, C…282    for (let i = 0; i < nSegments; i++) {283      const kind = rng.int(0, 2);284      let cond: { text: string; ok: boolean };285      if (kind === 0) {286        const usePresent = rng.next() < 0.5;287        const f = usePresent && present.length ? rng.pick(present) : rng.pick(absent);288        cond = { text: `test -f ${f}`, ok: present.includes(f) };289      } else if (kind === 1) {290        const useIn = rng.next() < 0.5;291        const w = useIn ? rng.pick(inHaystack) : rng.pick(notInHaystack);292        cond = { text: `grep -q ${w} notes.txt`, ok: inHaystack.includes(w) };293      } else {294        const ok = rng.next() < 0.5;295        cond = { text: ok ? 'true' : 'false', ok };296      }297      const thenEcho = String.fromCharCode(letter++);298      const elseEcho = String.fromCharCode(letter++);299      prims.push({ ...cond }, { text: `echo ${thenEcho}`, ok: true, echo: thenEcho }, { text: `echo ${elseEcho}`, ok: true, echo: elseEcho });300    }301302    // Build chain: cond && echo X || echo Y ; repeated (separate statements303    // joined with ';' so each triple is independent — unambiguous semantics).304    const statements: string[] = [];305    const printed: string[] = [];306    let lastExit = 0;307    for (let i = 0; i < prims.length; i += 3) {308      const cond = prims[i]!;309      const thenE = prims[i + 1]!;310      const elseE = prims[i + 2]!;311      statements.push(`${cond.text} && ${thenE.text} || ${elseE.text}`);312      // semantics: A && B || C — C runs if A fails OR B fails; echo never fails.313      if (cond.ok) {314        printed.push(thenE.echo!);315        lastExit = 0;316      } else {317        printed.push(elseE.echo!);318        lastExit = 0; // echo succeeds319      }320    }321    // Final statement without fallback → determines a non-trivial exit code.322    const finalOk = rng.next() < 0.5;323    const f = finalOk && present.length ? present[0]! : absent[0]!;324    const finalIsOk = present.includes(f);325    statements.push(`test -f ${f} && echo Z`);326    if (finalIsOk) {327      printed.push('Z');328      lastExit = 0;329    } else {330      lastExit = 1;331    }332333    const script = statements.join('\n');334    const answer = [...printed, `exit:${lastExit}`].join('\n');335336    const prompt = [337      'A POSIX shell session in a directory containing ONLY these files:',338      '',339      '```',340      [...present, 'notes.txt'].sort(byteCmp).join('\n'),341      '```',342      '',343      `\`notes.txt\` contains exactly the words: ${inHaystack.join(', ')} (one per line). No other files exist.`,344      '',345      'These statements run in order:',346      '',347      '```sh',348      script,349      '```',350      '',351      'Predict the terminal output: every line printed, in order, then a final line `exit:<N>` where N is the exit status of the LAST statement. ' +352        'Remember: `A && B || C` runs C whenever A fails (it is not a strict if/else); `test -f` succeeds only if the file exists; `grep -q` succeeds only if the word is present.',353      '',354      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,355    ].join('\n');356357    return {358      templateId: this.id,359      domain: this.domain,360      prompt,361      answerKey: answer,362      grading: 'lines' as const,363      perturbSeed,364    };365  },366};367