spb/llmindex Public

1/**2 * llmindex.io — code item templates (program-trace prediction, perturbed constants)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { ANSWER_FORMAT_INSTRUCTIONS } from '../answer';8import type { ItemTemplate } from '../types';910export const codeTracePython: ItemTemplate = {11  id: 'code.trace.python-v1',12  domain: 'code',13  description: 'Predict the printed output of a short Python loop with perturbed constants.',14  paramSpace: 15 * 9 * 20 * 30,15  render(rng, perturbSeed) {16    const start = rng.int(1, 15);17    const step = rng.int(2, 9);18    const limit = rng.int(30, 120);19    const divisor = rng.int(3, 7);20    const program = [21      `total = 0`,22      `v = ${start}`,23      `while total + v <= ${limit}:`,24      `    if v % ${divisor} != 0:`,25      `        total += v`,26      `    v += ${step}`,27      `print(total)`,28    ].join('\n');29    // Recompute exactly as the program executes (single source of truth).30    let t = 0;31    let x = start;32    while (t + x <= limit) {33      if (x % divisor !== 0) t += x;34      x += step;35    }36    const leads = [37      'What does this Python program print?',38      'Trace the following Python code and give its exact output.',39      'Execute this Python snippet mentally. What is printed?',40    ];41    return {42      templateId: this.id,43      domain: this.domain,44      prompt: `${rng.pick(leads)}\n\n\`\`\`python\n${program}\n\`\`\`\n\n${ANSWER_FORMAT_INSTRUCTIONS}`,45      answerKey: String(t),46      grading: 'numeric',47      perturbSeed,48    };49  },50};5152export const codeTraceNested: ItemTemplate = {53  id: 'code.trace.nested-v1',54  domain: 'code',55  description:56    'Trace nested loops with break/continue and an accumulator over generated constants — state tracking under control-flow interruptions.',57  paramSpace: 9 ** 5 * 4 ** 3,58  render(rng, perturbSeed) {59    const outer = rng.int(4, 7);60    const inner = rng.int(4, 7);61    const skipMod = rng.int(2, 4);62    const breakAt = rng.int(3, Math.max(3, inner - 1));63    const mult = rng.int(2, 5);64    // Simulate exactly the printed program.65    let total = 0;66    for (let i = 1; i <= outer; i++) {67      for (let j = 1; j <= inner; j++) {68        if (j === breakAt && i % 2 === 0) break;69        if ((i + j) % skipMod === 0) continue;70        total += i * mult + j;71      }72    }73    const program = [74      `total = 0`,75      `for i in range(1, ${outer + 1}):`,76      `    for j in range(1, ${inner + 1}):`,77      `        if j == ${breakAt} and i % 2 == 0:`,78      `            break`,79      `        if (i + j) % ${skipMod} == 0:`,80      `            continue`,81      `        total += i * ${mult} + j`,82      `print(total)`,83    ].join('\n');84    return {85      templateId: this.id,86      domain: this.domain,87      prompt: `Trace this Python program exactly. What does it print?\n\n\`\`\`python\n${program}\n\`\`\`\n\n${ANSWER_FORMAT_INSTRUCTIONS}`,88      answerKey: String(total),89      grading: 'numeric',90      perturbSeed,91    };92  },93};9495export const codeTraceJs: ItemTemplate = {96  id: 'code.trace.js-v1',97  domain: 'code',98  description: 'Predict the result of a JavaScript array pipeline with perturbed constants.',99  paramSpace: 8 * 12 * 10 * 6,100  render(rng, perturbSeed) {101    const len = rng.int(6, 12);102    const mult = rng.int(2, 7);103    const offset = rng.int(1, 10);104    const mod = rng.int(2, 5);105    const arr = Array.from({ length: len }, (_, i) => i + offset);106    const result = arr107      .map((n) => n * mult)108      .filter((n) => n % mod === 0)109      .reduce((a, b) => a + b, 0);110    const program = [111      `const arr = [${arr.join(', ')}];`,112      `const out = arr`,113      `  .map(n => n * ${mult})`,114      `  .filter(n => n % ${mod} === 0)`,115      `  .reduce((a, b) => a + b, 0);`,116      `console.log(out);`,117    ].join('\n');118    const leads = [119      'What does this JavaScript program log?',120      'Evaluate the following JavaScript. What number is logged to the console?',121    ];122    return {123      templateId: this.id,124      domain: this.domain,125      prompt: `${rng.pick(leads)}\n\n\`\`\`js\n${program}\n\`\`\`\n\n${ANSWER_FORMAT_INSTRUCTIONS}`,126      answerKey: String(result),127      grading: 'numeric',128      perturbSeed,129    };130  },131};132