/** * llmindex.io — terminal item templates: home-made shell simulation, deterministic ground truth * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * Original terminal-competence benchmark: no shell ever executes. A closed, * unambiguous POSIX subset (byte-order C-locale sort, integer-only awk forms, * fixed-string grep, no locale/format-dependent constructs) is simulated in * TypeScript; the model predicts exact output / final file tree / execution * traces, graded by exact match. Seeded generation of file contents and * pipelines makes memorization worthless. */ import { BLOCK_ANSWER_FORMAT_INSTRUCTIONS } from '../answer'; import type { Rng } from '../rng'; import type { ItemTemplate } from '../types'; /* ----------------------------- shared simulator ----------------------------- */ /** Byte-wise (C-locale) string comparison — ASCII-only data by construction. */ const byteCmp = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); type Lines = string[]; interface PipeStage { text: string; fn: (input: Lines) => Lines; } /* ------------------------------------------------------------------- * * Shape A: pipeline output prediction over a generated CSV * * ------------------------------------------------------------------- */ const DEPTS = ['sales', 'eng', 'ops', 'hr', 'legal'] as const; const FIRST = ['ana', 'bo', 'cy', 'dev', 'eli', 'fay', 'gus', 'hal', 'ivy', 'jon', 'kim', 'lou', 'max', 'ned', 'oli', 'pam'] as const; export const terminalPipeline: ItemTemplate = { id: 'terminal.pipeline.predict-v1', domain: 'terminal', description: 'Predict the exact stdout of a 3-5 stage pipeline (grep/cut/sort/head/tail/awk subset) over a generated CSV; byte-order vs numeric sort traps included.', paramSpace: 16 ** 10 * 5 ** 10 * 4 ** 5, render(rng: Rng, perturbSeed: string) { // Generate CSV rows: name,dept,units,score — names unique, ASCII lowercase. const nRows = rng.int(9, 14); const names = rng.shuffle(FIRST).slice(0, nRows); const rows = names.map((name) => { const dept = rng.pick(DEPTS); const units = rng.int(3, 120); // 1-3 digits → numeric-vs-byte sort trap const score = rng.int(10, 99); return `${name},${dept},${units},${score}`; }); // Build a pipeline of 3-5 stages from a closed, unambiguous set. const targetDept = rng.pick([...new Set(rows.map((r) => r.split(',')[1]!))]); const stages: PipeStage[] = []; stages.push({ text: `grep -F ',${targetDept},' people.csv`, fn: (input) => input.filter((l) => l.includes(`,${targetDept},`)), }); const variant = rng.int(0, 3); if (variant === 0) { // numeric sort on units, take top rows const n = rng.int(2, 3); stages.push({ text: `sort -t, -k3,3n`, fn: (input) => [...input].sort((a, b) => Number(a.split(',')[2]) - Number(b.split(',')[2]) || byteCmp(a, b)), }); stages.push({ text: `tail -n ${n}`, fn: (input) => input.slice(-n) }); } else if (variant === 1) { // byte-order sort on the whole line (the classic 100 < 9 trap), head const n = rng.int(2, 3); stages.push({ text: `cut -d, -f1,3`, fn: (input) => input.map((l) => l.split(',').filter((_, i) => i === 0 || i === 2).join(',')), }); stages.push({ text: `sort`, fn: (input) => [...input].sort(byteCmp) }); stages.push({ text: `head -n ${n}`, fn: (input) => input.slice(0, n) }); } else if (variant === 2) { // awk integer aggregation → single number stages.push({ text: `awk -F, '{ s += $3 } END { print s }'`, fn: (input) => [String(input.reduce((s, l) => s + Number(l.split(',')[2]), 0))], }); } else { // awk filter + count via grep -c style END counter const cutoff = rng.int(40, 80); stages.push({ text: `awk -F, '$4 > ${cutoff} { n += 1 } END { print n }'`, fn: (input) => [String(input.filter((l) => Number(l.split(',')[3]) > cutoff).length)], }); } let out: Lines = rows; for (const s of stages) out = s.fn(out); const pipeline = stages.map((s) => s.text).join(' | '); const prompt = [ 'A POSIX shell session (LC_ALL=C). The file `people.csv` contains exactly these lines (columns: name,dept,units,score):', '', '```', rows.join('\n'), '```', '', 'What is the EXACT stdout of this command?', '', '```sh', pipeline, '```', '', 'Notes: plain `sort` compares bytes (so "100" sorts before "9"); `sort -k3,3n` compares field 3 numerically.', '', BLOCK_ANSWER_FORMAT_INSTRUCTIONS, ].join('\n'); return { templateId: this.id, domain: this.domain, prompt, answerKey: out.join('\n'), grading: 'lines' as const, perturbSeed, }; }, }; /* ------------------------------------------------------------------- * * Shape B: file-tree prediction after mkdir/mv/cp/rm/touch + cd chain * * ------------------------------------------------------------------- */ const DIR_NAMES = ['src', 'docs', 'build', 'assets', 'logs', 'conf'] as const; const FILE_STEMS = ['main', 'util', 'index', 'setup', 'notes', 'report', 'draft', 'todo'] as const; const EXTS = ['txt', 'md', 'log', 'cfg'] as const; interface VirtualFs { files: Set; // absolute paths like /proj/src/main.txt dirs: Set; // absolute dir paths cwd: string; } export const terminalTree: ItemTemplate = { id: 'terminal.fs.tree-v1', domain: 'terminal', description: 'Track a virtual file tree through a sequence of mkdir/touch/mv/cp/rm with relative paths and cd; output the sorted final file list.', paramSpace: 6 ** 3 * 8 ** 5 * 4 ** 5 * 6 ** 6, render(rng: Rng, perturbSeed: string) { const fs: VirtualFs = { files: new Set(), dirs: new Set(['/proj']), cwd: '/proj' }; const dirs = rng.shuffle(DIR_NAMES).slice(0, 3); for (const d of dirs) fs.dirs.add(`/proj/${d}`); const stems = rng.shuffle(FILE_STEMS).slice(0, 5); for (const [i, stem] of stems.entries()) { const dir = i < 2 ? '/proj' : `/proj/${rng.pick(dirs)}`; fs.files.add(`${dir}/${stem}.${rng.pick(EXTS)}`); } const initialListing = [...fs.files].sort(byteCmp); const script: string[] = []; const nOps = rng.int(6, 8); for (let i = 0; i < nOps; i++) { const op = rng.int(0, 4); const fileArr = [...fs.files]; if (op === 0 && fileArr.length > 1) { // mv file → other dir (move) or rename in place const f = rng.pick(fileArr); if (rng.next() < 0.5) { const destDir = rng.pick([...fs.dirs]); const dest = `${destDir}/${f.split('/').pop()}`; script.push(`mv ${rel(fs.cwd, f)} ${rel(fs.cwd, destDir)}/`); fs.files.delete(f); fs.files.add(dest); } else { const newName = `${rng.pick(FILE_STEMS)}-${rng.int(1, 9)}.${rng.pick(EXTS)}`; const dest = `${f.split('/').slice(0, -1).join('/')}/${newName}`; script.push(`mv ${rel(fs.cwd, f)} ${rel(fs.cwd, dest)}`); fs.files.delete(f); fs.files.add(dest); } } else if (op === 1) { const destDir = rng.pick([...fs.dirs]); const f = rng.pick(fileArr); const dest = `${destDir}/${f.split('/').pop()}`; if (dest !== f) { script.push(`cp ${rel(fs.cwd, f)} ${rel(fs.cwd, destDir)}/`); fs.files.add(dest); } } else if (op === 2 && fileArr.length > 3) { const f = rng.pick(fileArr); script.push(`rm ${rel(fs.cwd, f)}`); fs.files.delete(f); } else if (op === 3) { const d = `${rng.pick([...fs.dirs])}/${rng.pick(DIR_NAMES)}-${rng.int(1, 9)}`; if (!fs.dirs.has(d)) { script.push(`mkdir -p ${rel(fs.cwd, d)}`); fs.dirs.add(d); } } else { const dir = rng.pick([...fs.dirs]); const f = `${dir}/${rng.pick(FILE_STEMS)}-${rng.int(1, 9)}.${rng.pick(EXTS)}`; if (!fs.files.has(f)) { script.push(`touch ${rel(fs.cwd, f)}`); fs.files.add(f); } } // occasionally change directory (forces relative-path tracking) if (rng.next() < 0.3) { const d = rng.pick([...fs.dirs]); script.push(`cd ${rel(fs.cwd, d) || '.'}`); fs.cwd = d; } } function rel(cwd: string, abs: string): string { if (abs === cwd) return '.'; if (abs.startsWith(cwd + '/')) return abs.slice(cwd.length + 1); // walk up from cwd to root then down — keep it simple and unambiguous const up = cwd.split('/').filter(Boolean).length; return '../'.repeat(up) + abs.replace(/^\//, ''); } const finalListing = [...fs.files].sort(byteCmp); const prompt = [ 'A POSIX shell session starts in `/proj`. The tree initially contains these FILES (directories exist as implied, plus empty dirs ' + dirs.map((d) => `\`/proj/${d}\``).join(', ') + '):', '', '```', initialListing.join('\n'), '```', '', 'These commands run in order (all succeed; `mv x dir/` moves into the directory; paths are relative to the CURRENT working directory, which `cd` changes):', '', '```sh', script.join('\n'), '```', '', 'List every file (absolute paths) that exists afterwards, one per line, sorted in byte order (C locale). Do not list directories.', '', BLOCK_ANSWER_FORMAT_INSTRUCTIONS, ].join('\n'); return { templateId: this.id, domain: this.domain, prompt, answerKey: finalListing.join('\n'), grading: 'lines' as const, perturbSeed, }; }, }; /* ------------------------------------------------------------------- * * Shape C: && / || short-circuit execution-trace prediction * * ------------------------------------------------------------------- */ export const terminalExitChain: ItemTemplate = { id: 'terminal.exit.chain-v1', domain: 'terminal', description: 'Predict which echo statements run and the final exit status of a && / || chain over test -f / grep -q primitives with known outcomes.', paramSpace: 2 ** 8 * 8 ** 4 * 26 ** 2, render(rng: Rng, perturbSeed: string) { // Known world: files that exist + a haystack file with known content. const present = ['app.txt', 'data.txt'].filter(() => rng.next() < 0.8); const absent = ['ghost.txt', 'tmp.txt']; const words = ['amber', 'basil', 'coral', 'dune']; const inHaystack = rng.shuffle(words).slice(0, 2); const notInHaystack = words.filter((w) => !inHaystack.includes(w)); interface Prim { text: string; ok: boolean; echo?: string; } const prims: Prim[] = []; const nSegments = rng.int(3, 4); let letter = 65; // A, B, C… for (let i = 0; i < nSegments; i++) { const kind = rng.int(0, 2); let cond: { text: string; ok: boolean }; if (kind === 0) { const usePresent = rng.next() < 0.5; const f = usePresent && present.length ? rng.pick(present) : rng.pick(absent); cond = { text: `test -f ${f}`, ok: present.includes(f) }; } else if (kind === 1) { const useIn = rng.next() < 0.5; const w = useIn ? rng.pick(inHaystack) : rng.pick(notInHaystack); cond = { text: `grep -q ${w} notes.txt`, ok: inHaystack.includes(w) }; } else { const ok = rng.next() < 0.5; cond = { text: ok ? 'true' : 'false', ok }; } const thenEcho = String.fromCharCode(letter++); const elseEcho = String.fromCharCode(letter++); prims.push({ ...cond }, { text: `echo ${thenEcho}`, ok: true, echo: thenEcho }, { text: `echo ${elseEcho}`, ok: true, echo: elseEcho }); } // Build chain: cond && echo X || echo Y ; repeated (separate statements // joined with ';' so each triple is independent — unambiguous semantics). const statements: string[] = []; const printed: string[] = []; let lastExit = 0; for (let i = 0; i < prims.length; i += 3) { const cond = prims[i]!; const thenE = prims[i + 1]!; const elseE = prims[i + 2]!; statements.push(`${cond.text} && ${thenE.text} || ${elseE.text}`); // semantics: A && B || C — C runs if A fails OR B fails; echo never fails. if (cond.ok) { printed.push(thenE.echo!); lastExit = 0; } else { printed.push(elseE.echo!); lastExit = 0; // echo succeeds } } // Final statement without fallback → determines a non-trivial exit code. const finalOk = rng.next() < 0.5; const f = finalOk && present.length ? present[0]! : absent[0]!; const finalIsOk = present.includes(f); statements.push(`test -f ${f} && echo Z`); if (finalIsOk) { printed.push('Z'); lastExit = 0; } else { lastExit = 1; } const script = statements.join('\n'); const answer = [...printed, `exit:${lastExit}`].join('\n'); const prompt = [ 'A POSIX shell session in a directory containing ONLY these files:', '', '```', [...present, 'notes.txt'].sort(byteCmp).join('\n'), '```', '', `\`notes.txt\` contains exactly the words: ${inHaystack.join(', ')} (one per line). No other files exist.`, '', 'These statements run in order:', '', '```sh', script, '```', '', 'Predict the terminal output: every line printed, in order, then a final line `exit:` where N is the exit status of the LAST statement. ' + 'Remember: `A && B || C` runs C whenever A fails (it is not a strict if/else); `test -f` succeeds only if the file exists; `grep -q` succeeds only if the word is present.', '', BLOCK_ANSWER_FORMAT_INSTRUCTIONS, ].join('\n'); return { templateId: this.id, domain: this.domain, prompt, answerKey: answer, grading: 'lines' as const, perturbSeed, }; }, };