/** * llmindex.io — agentic-under-context-load template: tool use with a buried-facts ledger * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * Measures agentic performance under CONTEXT LENGTH LOAD: the model must scan * a large generated ledger (hundreds of near-miss decoy records), select the * few records matching a compound policy, derive tool arguments from them, and * emit the exact call sequence. Record count is the load knob; decoys are * semantically adjacent (same customer/other region, same region/other status) * so skimming fails. Ground truth from a deterministic simulator. */ import { BLOCK_ANSWER_FORMAT_INSTRUCTIONS, canonicalJson } from '../answer'; import type { Rng } from '../rng'; import type { ItemTemplate } from '../types'; const CUSTOMERS = ['acme', 'birch', 'cobalt', 'dorian', 'ember', 'fulton', 'gale', 'harbor', 'ionic', 'juno'] as const; const REGIONS = ['east', 'west', 'north', 'south'] as const; const ITEMS = ['valve', 'rotor', 'panel', 'cable', 'sensor', 'frame', 'pump', 'gasket'] as const; const STATUSES = ['pending', 'paid', 'shipped', 'held'] as const; interface Order { id: number; customer: string; region: string; item: string; qty: number; status: string; } export const agenticContextLoad: ItemTemplate = { id: 'agentic.tools.context-load-v1', domain: 'agentic', description: 'Apply a compound order-processing policy over a 120-300 row generated ledger dense with near-miss decoys; call sequence must be exact — context-length load is the difficulty knob.', paramSpace: 10 * 4 * 4 * 8 ** 3 * 10 ** 6, render(rng: Rng, perturbSeed: string) { const nRecords = rng.int(120, 300); const targetCustomer = rng.pick(CUSTOMERS); const targetRegion = rng.pick(REGIONS); const targetStatus = 'pending'; const qtyThreshold = rng.int(40, 70); // Generate the ledger with guaranteed near-miss density: for each true // match, several decoys differing in exactly one predicate. const orders: Order[] = []; let nextId = 1000 + rng.int(0, 500); const addOrder = (o: Omit): void => { orders.push({ id: nextId, ...o }); nextId += rng.int(1, 7); }; const nMatches = rng.int(3, 5); for (let i = 0; i < nMatches; i++) { addOrder({ customer: targetCustomer, region: targetRegion, item: rng.pick(ITEMS), qty: rng.int(10, 99), status: targetStatus, }); // adjacent decoys addOrder({ customer: targetCustomer, region: rng.pick(REGIONS.filter((r) => r !== targetRegion)), item: rng.pick(ITEMS), qty: rng.int(10, 99), status: targetStatus, }); addOrder({ customer: targetCustomer, region: targetRegion, item: rng.pick(ITEMS), qty: rng.int(10, 99), status: rng.pick(STATUSES.filter((s) => s !== targetStatus)), }); } while (orders.length < nRecords) { addOrder({ customer: rng.pick(CUSTOMERS), region: rng.pick(REGIONS), item: rng.pick(ITEMS), qty: rng.int(10, 99), status: rng.pick(STATUSES), }); } const ledger = rng.shuffle(orders); // Simulator: matches in ascending order id; branch on qty threshold. const matches = ledger .filter((o) => o.customer === targetCustomer && o.region === targetRegion && o.status === targetStatus) .sort((a, b) => a.id - b.id); const expected = matches.map((o) => o.qty > qtyThreshold ? { tool: 'restock', args: { item: o.item, qty: o.qty } } : { tool: 'cancel_order', args: { order_id: o.id } }, ); const tools = [ '- restock(item: string, qty: int) — reorders stock for a large pending order', '- cancel_order(order_id: int) — cancels a small pending order', // distractors '- ship_order(order_id: int) — ships a paid order (out of scope here)', '- refund(order_id: int, amount: int) — refunds a customer (out of scope here)', '- notify_customer(customer: string, message: string) — sends a notification (not required by this policy)', ].join('\n'); const ledgerLines = ledger .map((o) => `${o.id}|${o.customer}|${o.region}|${o.item}|${o.qty}|${o.status}`) .join('\n'); const prompt = [ 'You are an order-operations agent working strictly through tool calls.', '', 'TOOL CATALOG:', tools, '', `ORDER LEDGER (${ledger.length} records, format: id|customer|region|item|qty|status):`, '```', ledgerLines, '```', '', 'POLICY (apply exactly):', `- Consider ONLY orders where customer = "${targetCustomer}" AND region = "${targetRegion}" AND status = "${targetStatus}".`, `- Process those orders in ASCENDING order id.`, `- For each: if qty > ${qtyThreshold}, call restock(item, qty) with that order's item and qty; otherwise call cancel_order(order_id).`, '- Call nothing else. Every other record is irrelevant no matter how similar it looks.', '', 'Output the full ordered sequence of tool calls as a JSON array of {"tool": string, "args": object}.', '', BLOCK_ANSWER_FORMAT_INSTRUCTIONS, ].join('\n'); return { templateId: this.id, domain: this.domain, prompt, answerKey: canonicalJson(expected), grading: 'json' as const, perturbSeed, }; }, };