spb/llmindex Public

1/**2 * llmindex.io — agentic item templates: simulated tool-calling, deterministic ground truth3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Original, home-made agentic evaluation (inspired by the *ideas* behind8 * public agentic benchmarks — mock environments, state-based grading — but9 * with our own environments, rules and grading; nothing is copied).10 *11 * Each item presents a mock tool catalog (with distractor tools), a world12 * state, deterministic business rules, and a goal. The model must emit the13 * EXACT ordered JSON call sequence; a built-in simulator computes the unique14 * correct sequence, so grading is canonical-JSON equality — no judges.15 */16import { BLOCK_ANSWER_FORMAT_INSTRUCTIONS, canonicalJson } from '../answer';17import type { Rng } from '../rng';18import type { ItemTemplate } from '../types';1920interface ToolCall {21  tool: string;22  args: Record<string, string | number>;23}2425const OUTPUT_RULES =26  'Output the full ordered sequence of tool calls needed to accomplish the goal, as a JSON array ' +27  'of objects {"tool": string, "args": object}. Use exactly the tool and argument names from the ' +28  'catalog. Do not call any tool that is not required.';2930function catalogBlock(tools: Array<{ sig: string; desc: string }>): string {31  return tools.map((t) => `- ${t.sig} — ${t.desc}`).join('\n');32}3334/* ------------------------------------------------------------------ *35 * Shape A: support-desk triage (policy routing with distractor tools) *36 * ------------------------------------------------------------------ */3738const INCIDENT_KINDS = [39  { kind: 'payments', words: ['refund double-charged', 'card declined at checkout', 'invoice total wrong'] },40  { kind: 'auth', words: ['cannot reset password', 'locked out after 2FA change', 'SSO loop on login'] },41  { kind: 'data', words: ['export file corrupted', 'dashboard shows stale numbers', 'records missing after import'] },42  { kind: 'infra', words: ['API latency spikes', 'webhooks not delivered', 'uploads failing intermittently'] },43] as const;4445const AGENT_NAMES = ['rivera', 'chen', 'okafor', 'dubois', 'tanaka', 'silva', 'novak', 'haddad'] as const;4647export const agenticTriage: ItemTemplate = {48  id: 'agentic.tools.triage-v1',49  domain: 'agentic',50  description:51    'Route generated incidents through a ticket system under escalation/skill policies; unique correct call sequence, distractor tools present.',52  paramSpace: 4 ** 4 * 8 ** 3 * 10 ** 4 * 24,53  render(rng: Rng, perturbSeed: string) {54    const nIncidents = rng.int(3, 4);55    const agents = rng.shuffle(AGENT_NAMES).slice(0, 3);56    const kinds = rng.shuffle(INCIDENT_KINDS).slice(0, 3);57    // Skill table: each of the 3 kinds handled by exactly one agent.58    const skills = kinds.map((k, i) => ({ kind: k.kind, agent: agents[i]! }));59    const escalateAt = rng.int(7, 9);6061    interface Incident {62      desc: string;63      kind: string;64      priority: number;65      duplicateOf?: number;66    }67    const incidents: Incident[] = [];68    for (let i = 0; i < nIncidents; i++) {69      const k = kinds[i % kinds.length]!;70      incidents.push({71        desc: rng.pick(k.words),72        kind: k.kind,73        priority: rng.int(2, 9),74      });75    }76    // One incident (never the first) is a duplicate of an earlier one.77    const dupIdx = rng.int(1, nIncidents - 1);78    const origIdx = rng.int(0, dupIdx - 1);79    incidents[dupIdx] = { ...incidents[origIdx]!, duplicateOf: origIdx };8081    // Simulator: unique correct sequence under the stated policy.82    const expected: ToolCall[] = [];83    incidents.forEach((inc, i) => {84      const id = `TCK-${i + 1}`;85      expected.push({ tool: 'create_ticket', args: { title: inc.desc, priority: inc.priority } });86      if (inc.duplicateOf !== undefined) {87        expected.push({88          tool: 'close_ticket',89          args: { ticket_id: id, resolution: `duplicate of TCK-${inc.duplicateOf + 1}` },90        });91        return;92      }93      if (inc.priority >= escalateAt) expected.push({ tool: 'escalate', args: { ticket_id: id } });94      const agent = skills.find((s) => s.kind === inc.kind)!.agent;95      expected.push({ tool: 'assign', args: { ticket_id: id, agent } });96    });9798    const tools = [99      { sig: 'create_ticket(title: string, priority: int)', desc: 'opens a ticket; IDs are assigned sequentially: the 1st created ticket is "TCK-1", the 2nd "TCK-2", etc.' },100      { sig: 'assign(ticket_id: string, agent: string)', desc: 'assigns an open ticket to an agent' },101      { sig: 'escalate(ticket_id: string)', desc: 'marks a ticket as escalated' },102      { sig: 'close_ticket(ticket_id: string, resolution: string)', desc: 'closes a ticket with a resolution note' },103      // distractors — never needed104      { sig: 'send_email(to: string, body: string)', desc: 'sends an email (not part of the triage policy)' },105      { sig: 'archive_ticket(ticket_id: string)', desc: 'archives a closed ticket (nightly job does this automatically)' },106      { sig: 'set_reminder(ticket_id: string, hours: int)', desc: 'sets a follow-up reminder' },107    ];108109    const skillLines = skills.map((s) => `- ${s.kind} → ${s.agent}`).join('\n');110    const incidentLines = incidents111      .map((inc, i) => `${i + 1}. "${inc.desc}" (category: ${inc.kind}, priority ${inc.priority})`)112      .join('\n');113114    const prompt = [115      'You operate a support desk strictly through tool calls.',116      '',117      'TOOL CATALOG:',118      catalogBlock(tools),119      '',120      'ROUTING POLICY (apply exactly, in this order, for each incident, processing incidents in the order listed):',121      '1. Create a ticket for the incident (title = the incident text verbatim, priority as given).',122      `2. If the incident is an exact duplicate of an earlier incident in this list, close its ticket immediately with resolution "duplicate of <ID of the earlier ticket>" and do nothing else for it.`,123      `3. Otherwise, if priority ≥ ${escalateAt}, escalate the ticket BEFORE assigning it.`,124      '4. Assign the ticket to the agent responsible for its category.',125      '',126      'CATEGORY → AGENT:',127      skillLines,128      '',129      'INCIDENTS:',130      incidentLines,131      '',132      OUTPUT_RULES,133      '',134      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,135    ].join('\n');136137    return {138      templateId: this.id,139      domain: this.domain,140      prompt,141      answerKey: canonicalJson(expected),142      grading: 'json' as const,143      perturbSeed,144    };145  },146};147148/* --------------------------------------------------------------- *149 * Shape B: treasury ledger (stateful arithmetic + conditional flow) *150 * --------------------------------------------------------------- */151152const ACCOUNT_NAMES = ['alpha', 'bravo', 'delta', 'echo', 'kilo', 'lima', 'oscar', 'tango'] as const;153154export const agenticLedger: ItemTemplate = {155  id: 'agentic.tools.ledger-v1',156  domain: 'agentic',157  description:158    'Execute payment instructions over account balances; overdrafts must be pre-funded from reserve with the exact shortfall — requires running-state arithmetic.',159  paramSpace: 8 ** 3 * 900 ** 3 * 500 ** 4,160  render(rng: Rng, perturbSeed: string) {161    const accounts = rng.shuffle(ACCOUNT_NAMES).slice(0, 3);162    const balances = new Map<string, number>();163    for (const a of accounts) balances.set(a, rng.int(120, 900));164    const nPayments = rng.int(4, 5);165166    interface Payment {167      from: string;168      to: string;169      amount: number;170    }171    const payments: Payment[] = [];172    for (let i = 0; i < nPayments; i++) {173      const from = rng.pick(accounts);174      let to = rng.pick(accounts);175      while (to === from) to = rng.pick(accounts);176      payments.push({ from, to, amount: rng.int(80, 600) });177    }178179    // Simulator: transfers in order; shortfall → top_up_from_reserve first.180    const expected: ToolCall[] = [];181    const state = new Map(balances);182    for (const p of payments) {183      const bal = state.get(p.from)!;184      if (bal < p.amount) {185        const shortfall = p.amount - bal;186        expected.push({ tool: 'top_up_from_reserve', args: { account: p.from, amount: shortfall } });187        state.set(p.from, bal + shortfall);188      }189      expected.push({ tool: 'transfer', args: { from: p.from, to: p.to, amount: p.amount } });190      state.set(p.from, state.get(p.from)! - p.amount);191      state.set(p.to, state.get(p.to)! + p.amount);192    }193194    const tools = [195      { sig: 'transfer(from: string, to: string, amount: int)', desc: 'moves funds between accounts; FAILS if it would overdraw the source' },196      { sig: 'top_up_from_reserve(account: string, amount: int)', desc: 'adds funds to an account from the corporate reserve' },197      // distractors198      { sig: 'get_balance(account: string)', desc: 'reads a balance (you already have all balances below — reads are unnecessary and forbidden)' },199      { sig: 'freeze_account(account: string)', desc: 'compliance freeze (not part of this task)' },200      { sig: 'convert_currency(account: string, currency: string)', desc: 'FX conversion (all amounts are already in USD)' },201    ];202203    const balanceLines = accounts.map((a) => `- ${a}: $${balances.get(a)}`).join('\n');204    const paymentLines = payments205      .map((p, i) => `${i + 1}. pay $${p.amount} from "${p.from}" to "${p.to}"`)206      .join('\n');207208    const prompt = [209      'You are a treasury agent operating strictly through tool calls.',210      '',211      'TOOL CATALOG:',212      catalogBlock(tools),213      '',214      'OPENING BALANCES:',215      balanceLines,216      '',217      'PAYMENT INSTRUCTIONS (execute in exactly this order):',218      paymentLines,219      '',220      'RULES:',221      '- transfer() fails on overdraft. If a payment would overdraw its source account at the moment of execution, first call top_up_from_reserve() on the source with EXACTLY the shortfall (no more, no less), then execute the transfer.',222      '- Track balances as they change: earlier payments affect later ones.',223      '- Never call tools that are not needed.',224      '',225      OUTPUT_RULES,226      '',227      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,228    ].join('\n');229230    return {231      templateId: this.id,232      domain: this.domain,233      prompt,234      answerKey: canonicalJson(expected),235      grading: 'json' as const,236      perturbSeed,237    };238  },239};240241/* ----------------------------------------------------------- *242 * Shape C: deployment pipeline (dependency-ordered operations)  *243 * ----------------------------------------------------------- */244245const SERVICE_NAMES = ['gateway', 'billing', 'search', 'notifier', 'reports', 'auth-svc'] as const;246247export const agenticDeploy: ItemTemplate = {248  id: 'agentic.tools.deploy-v1',249  domain: 'agentic',250  description:251    'Deploy services respecting a dependency DAG and health-gate policy; correct topological order with deterministic tie-breaking.',252  paramSpace: 6 ** 4 * 2 ** 6 * 24,253  render(rng: Rng, perturbSeed: string) {254    const services = rng.shuffle(SERVICE_NAMES).slice(0, 4);255    // Build a random DAG over the 4 services: edges only from earlier to later256    // in a hidden topological order.257    const order = rng.shuffle(services);258    const deps = new Map<string, string[]>();259    for (const s of services) deps.set(s, []);260    for (let i = 1; i < order.length; i++) {261      const nDeps = rng.int(1, Math.min(2, i));262      const chosen = rng.shuffle(order.slice(0, i)).slice(0, nDeps);263      deps.set(order[i]!, chosen.sort());264    }265    const flaky = rng.pick(services); // needs a health check after deploy266267    // Simulator: repeated passes; deploy every service whose deps are all268    // deployed, in ALPHABETICAL order within a pass (stated tie-break rule).269    const expected: ToolCall[] = [];270    const deployed = new Set<string>();271    while (deployed.size < services.length) {272      const ready = services273        .filter((s) => !deployed.has(s) && deps.get(s)!.every((d) => deployed.has(d)))274        .sort();275      for (const s of ready) {276        expected.push({ tool: 'deploy', args: { service: s } });277        if (s === flaky) expected.push({ tool: 'health_check', args: { service: s } });278        deployed.add(s);279      }280    }281282    const tools = [283      { sig: 'deploy(service: string)', desc: 'deploys a service; FAILS if any dependency is not yet deployed' },284      { sig: 'health_check(service: string)', desc: 'runs a post-deploy health probe' },285      // distractors286      { sig: 'rollback(service: string)', desc: 'reverts a bad deploy (nothing fails in this scenario)' },287      { sig: 'scale(service: string, replicas: int)', desc: 'changes replica count (out of scope)' },288      { sig: 'restart(service: string)', desc: 'restarts a service (out of scope)' },289    ];290291    const depLines = services292      .map((s) => `- ${s}: ${deps.get(s)!.length ? deps.get(s)!.join(', ') : '(none)'}`)293      .join('\n');294295    const prompt = [296      'You are a release agent operating strictly through tool calls.',297      '',298      'TOOL CATALOG:',299      catalogBlock(tools),300      '',301      'SERVICES AND THEIR DEPENDENCIES (a service can only be deployed after ALL its dependencies):',302      depLines,303      '',304      'POLICY:',305      '- Deploy in waves: in each wave, deploy every service whose dependencies are already deployed, in alphabetical order; repeat until all services are deployed.',306      `- The service "${flaky}" is flagged unstable: call health_check on it immediately after deploying it.`,307      '- Call nothing else.',308      '',309      OUTPUT_RULES,310      '',311      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,312    ].join('\n');313314    return {315      templateId: this.id,316      domain: this.domain,317      prompt,318      answerKey: canonicalJson(expected),319      grading: 'json' as const,320      perturbSeed,321    };322  },323};324