spb/llmindex Public

1/**2 * llmindex.io — agentic-under-context-load template: tool use with a buried-facts ledger3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Measures agentic performance under CONTEXT LENGTH LOAD: the model must scan8 * a large generated ledger (hundreds of near-miss decoy records), select the9 * few records matching a compound policy, derive tool arguments from them, and10 * emit the exact call sequence. Record count is the load knob; decoys are11 * semantically adjacent (same customer/other region, same region/other status)12 * so skimming fails. Ground truth from a deterministic simulator.13 */14import { BLOCK_ANSWER_FORMAT_INSTRUCTIONS, canonicalJson } from '../answer';15import type { Rng } from '../rng';16import type { ItemTemplate } from '../types';1718const CUSTOMERS = ['acme', 'birch', 'cobalt', 'dorian', 'ember', 'fulton', 'gale', 'harbor', 'ionic', 'juno'] as const;19const REGIONS = ['east', 'west', 'north', 'south'] as const;20const ITEMS = ['valve', 'rotor', 'panel', 'cable', 'sensor', 'frame', 'pump', 'gasket'] as const;21const STATUSES = ['pending', 'paid', 'shipped', 'held'] as const;2223interface Order {24  id: number;25  customer: string;26  region: string;27  item: string;28  qty: number;29  status: string;30}3132export const agenticContextLoad: ItemTemplate = {33  id: 'agentic.tools.context-load-v1',34  domain: 'agentic',35  description:36    'Apply a compound order-processing policy over a 120-300 row generated ledger dense with near-miss decoys; call sequence must be exact — context-length load is the difficulty knob.',37  paramSpace: 10 * 4 * 4 * 8 ** 3 * 10 ** 6,38  render(rng: Rng, perturbSeed: string) {39    const nRecords = rng.int(120, 300);40    const targetCustomer = rng.pick(CUSTOMERS);41    const targetRegion = rng.pick(REGIONS);42    const targetStatus = 'pending';43    const qtyThreshold = rng.int(40, 70);4445    // Generate the ledger with guaranteed near-miss density: for each true46    // match, several decoys differing in exactly one predicate.47    const orders: Order[] = [];48    let nextId = 1000 + rng.int(0, 500);49    const addOrder = (o: Omit<Order, 'id'>): void => {50      orders.push({ id: nextId, ...o });51      nextId += rng.int(1, 7);52    };5354    const nMatches = rng.int(3, 5);55    for (let i = 0; i < nMatches; i++) {56      addOrder({57        customer: targetCustomer,58        region: targetRegion,59        item: rng.pick(ITEMS),60        qty: rng.int(10, 99),61        status: targetStatus,62      });63      // adjacent decoys64      addOrder({65        customer: targetCustomer,66        region: rng.pick(REGIONS.filter((r) => r !== targetRegion)),67        item: rng.pick(ITEMS),68        qty: rng.int(10, 99),69        status: targetStatus,70      });71      addOrder({72        customer: targetCustomer,73        region: targetRegion,74        item: rng.pick(ITEMS),75        qty: rng.int(10, 99),76        status: rng.pick(STATUSES.filter((s) => s !== targetStatus)),77      });78    }79    while (orders.length < nRecords) {80      addOrder({81        customer: rng.pick(CUSTOMERS),82        region: rng.pick(REGIONS),83        item: rng.pick(ITEMS),84        qty: rng.int(10, 99),85        status: rng.pick(STATUSES),86      });87    }88    const ledger = rng.shuffle(orders);8990    // Simulator: matches in ascending order id; branch on qty threshold.91    const matches = ledger92      .filter((o) => o.customer === targetCustomer && o.region === targetRegion && o.status === targetStatus)93      .sort((a, b) => a.id - b.id);94    const expected = matches.map((o) =>95      o.qty > qtyThreshold96        ? { tool: 'restock', args: { item: o.item, qty: o.qty } }97        : { tool: 'cancel_order', args: { order_id: o.id } },98    );99100    const tools = [101      '- restock(item: string, qty: int) — reorders stock for a large pending order',102      '- cancel_order(order_id: int) — cancels a small pending order',103      // distractors104      '- ship_order(order_id: int) — ships a paid order (out of scope here)',105      '- refund(order_id: int, amount: int) — refunds a customer (out of scope here)',106      '- notify_customer(customer: string, message: string) — sends a notification (not required by this policy)',107    ].join('\n');108109    const ledgerLines = ledger110      .map((o) => `${o.id}|${o.customer}|${o.region}|${o.item}|${o.qty}|${o.status}`)111      .join('\n');112113    const prompt = [114      'You are an order-operations agent working strictly through tool calls.',115      '',116      'TOOL CATALOG:',117      tools,118      '',119      `ORDER LEDGER (${ledger.length} records, format: id|customer|region|item|qty|status):`,120      '```',121      ledgerLines,122      '```',123      '',124      'POLICY (apply exactly):',125      `- Consider ONLY orders where customer = "${targetCustomer}" AND region = "${targetRegion}" AND status = "${targetStatus}".`,126      `- Process those orders in ASCENDING order id.`,127      `- For each: if qty > ${qtyThreshold}, call restock(item, qty) with that order's item and qty; otherwise call cancel_order(order_id).`,128      '- Call nothing else. Every other record is irrelevant no matter how similar it looks.',129      '',130      'Output the full ordered sequence of tool calls as a JSON array of {"tool": string, "args": object}.',131      '',132      BLOCK_ANSWER_FORMAT_INSTRUCTIONS,133    ].join('\n');134135    return {136      templateId: this.id,137      domain: this.domain,138      prompt,139      answerKey: canonicalJson(expected),140      grading: 'json' as const,141      perturbSeed,142    };143  },144};145