1"""Executable task family: small, deterministic coding/data tasks whose correctness is checked by hidden tests.23Family 'exec': an agent receives (a) a source document describing a small dataset + a bug report or analysis request and4(b) a task; it must output a Python function body. Hidden tests (evaluator-only) run in a clean subprocess with a timeout.5The 'document' is the information the sender holds (table schema, bug report, expected behaviour); the receiver must6reconstruct the specification from the message. Ground truth is generated deterministically from a seed.7"""8from __future__ import annotations9import random, json, textwrap, subprocess, sys, tempfile, os10from dataclasses import dataclass, asdict1112COLS = ["region", "product", "units", "unit_price", "discount_pct", "returned", "shipped_days"]13REGIONS = ["Nord", "Sud", "Est", "Ouest", "Centre"]14PRODUCTS = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"]1516def make_table(rng: random.Random, n: int):17 rows = []18 for _ in range(n):19 rows.append({"region": rng.choice(REGIONS), "product": rng.choice(PRODUCTS), "units": rng.randint(1, 40),20 "unit_price": round(rng.uniform(2.5, 80.0), 2), "discount_pct": rng.choice([0, 0, 5, 10, 15, 25]),21 "returned": rng.random() < 0.12, "shipped_days": rng.randint(1, 14)})22 return rows2324def spec_variants(rng: random.Random, rows):25 """Return (spec_text, reference_fn_source, test_cases). Spec encodes exact business rules that must be transmitted."""26 thr_days = rng.randint(3, 9)27 min_units = rng.randint(2, 6)28 excl_region = rng.choice(REGIONS)29 rounding = rng.choice([0, 1, 2])30 tax = rng.choice([0.0, 5.0, 9.975, 14.975])31 spec = (f"Bug report BR-{rng.randint(1000,9999)}: the function net_revenue(rows) must compute the total net revenue of the "32 f"rows list (each row is a dict with keys {COLS}). Rules agreed with the client on {rng.randint(1,28)} {rng.choice(['March','June','September'])} 2031: "33 f"(1) exclude returned rows; (2) exclude rows whose region is '{excl_region}'; (3) exclude rows with fewer than {min_units} units; "34 f"(4) rows shipped in more than {thr_days} days receive an additional 5 percentage points of discount on top of discount_pct, capped at 50 %; "35 f"(5) line revenue = units * unit_price * (1 - total_discount/100); (6) apply a tax of {tax} % on the sum; "36 f"(7) return the result rounded to {rounding} decimals (Python round). The current implementation forgets rule (4) and rule (2).")37 ref = textwrap.dedent(f"""38 def net_revenue(rows):39 total = 0.040 for r in rows:41 if r['returned']:42 continue43 if r['region'] == '{excl_region}':44 continue45 if r['units'] < {min_units}:46 continue47 d = r['discount_pct'] + (5 if r['shipped_days'] > {thr_days} else 0)48 d = min(d, 50)49 total += r['units'] * r['unit_price'] * (1 - d / 100.0)50 total = total * (1 + {tax} / 100.0)51 return round(total, {rounding})52 """)53 ns = {}54 exec(ref, ns)55 tests = []56 for k in range(5):57 sub = [dict(r) for r in rng.sample(rows, k=min(len(rows), rng.randint(4, len(rows))))]58 tests.append({"input": sub, "expected": ns["net_revenue"](sub)})59 return spec, ref, tests, {"thr_days": thr_days, "min_units": min_units, "excl_region": excl_region, "rounding": rounding, "tax": tax}6061def generate_exec_task(seed: int, target_tokens: int = 2000) -> dict:62 rng = random.Random(10_000 + seed)63 rows = make_table(rng, rng.randint(12, 24))64 spec, ref, tests, params = spec_variants(rng, rows)65 sample_rows = json.dumps(rows[:6], indent=None)66 filler = []67 boiler = ["The ticketing system assigns severity on a four-level scale; this ticket is severity 2.",68 "All monetary values are in the client's reporting currency; no FX conversion is applied here.",69 "Unit tests live in the tests/ directory and are executed in CI on every push.",70 "The data engineering team refreshes the sales extract nightly at 02:00 local time.",71 "Code style follows the repository's formatter settings; do not reformat unrelated lines.",72 "Deployment to production requires a reviewer approval and a green pipeline.",73 "Historic tickets BR-1183 and BR-2047 are unrelated and remain closed.",74 "The data dictionary defines shipped_days as the number of calendar days from order to shipment."]75 n_fill = max(0, int(target_tokens * 4 / 95) - 12)76 filler = [rng.choice(boiler) for _ in range(n_fill)]77 doc = ("ENGINEERING TICKET\n\n" + spec + "\n\nSample rows (first 6 of the extract):\n" + sample_rows + "\n\n" +78 "\n".join(filler[: n_fill // 2]) + "\n\nThe existing (buggy) implementation is:\n\n" +79 "def net_revenue(rows):\n total = 0.0\n for r in rows:\n if r['returned']:\n continue\n"80 f" if r['units'] < {params['min_units']}:\n continue\n d = r['discount_pct']\n"81 " total += r['units'] * r['unit_price'] * (1 - d / 100.0)\n"82 f" total = total * (1 + {params['tax']} / 100.0)\n return round(total, {params['rounding']})\n\n" + "\n".join(filler[n_fill // 2:]))83 task = ("Rewrite the Python function net_revenue(rows) so that it implements ALL the agreed rules in the ticket. "84 "Output only Python code defining net_revenue; no explanations, no markdown fences.")85 return {"document_id": f"exec_{seed:05d}_{target_tokens}", "seed": seed, "target_tokens": target_tokens, "text": doc, "char_len": len(doc),86 "questions": [{"question_id": "q00_exec", "qtype": "exec", "question": task, "answer": ref, "answer_type": "exec", "tolerance": 0.0,87 "supporting_fact_ids": ["spec"], "answer_position_frac": 0.02, "aliases": []}],88 "facts": [{"fact_id": "spec", "entity": "net_revenue", "attribute": "rules", "value": json.dumps(params), "kind": "spec", "sentence": spec, "char_start": doc.find(spec), "char_end": doc.find(spec) + len(spec)}],89 "hidden_tests": tests}9091def run_hidden_tests(code: str, tests: list, timeout_s: float = 10.0) -> dict:92 """Execute candidate code in a clean subprocess (no network, no project imports) against hidden tests."""93 code = code.strip()94 if code.startswith("```"):95 code = code.strip("`")96 if code.startswith("python"):97 code = code[6:]98 harness = textwrap.dedent("""99 import json, sys, math100 src = open(sys.argv[1]).read(); tests = json.load(open(sys.argv[2]))101 ns = {}102 try:103 exec(src, ns)104 except Exception as e:105 print(json.dumps({"passed": 0, "total": len(tests), "error": "exec:" + type(e).__name__})); sys.exit(0)106 fn = ns.get('net_revenue')107 if fn is None:108 print(json.dumps({"passed": 0, "total": len(tests), "error": "no_function"})); sys.exit(0)109 p = 0; err = None110 for t in tests:111 try:112 out = fn([dict(r) for r in t['input']])113 if isinstance(out, (int, float)) and math.isclose(float(out), float(t['expected']), rel_tol=1e-9, abs_tol=1e-6):114 p += 1115 except Exception as e:116 err = type(e).__name__117 print(json.dumps({"passed": p, "total": len(tests), "error": err}))118 """)119 with tempfile.TemporaryDirectory() as d:120 cp = os.path.join(d, "cand.py"); tp = os.path.join(d, "tests.json"); hp = os.path.join(d, "h.py")121 open(cp, "w").write(code); json.dump(tests, open(tp, "w")); open(hp, "w").write(harness)122 try:123 r = subprocess.run([sys.executable, "-I", hp, cp, tp], capture_output=True, text=True, timeout=timeout_s, cwd=d)124 res = json.loads(r.stdout.strip().splitlines()[-1]) if r.stdout.strip() else {"passed": 0, "total": len(tests), "error": "no_output"}125 except subprocess.TimeoutExpired:126 res = {"passed": 0, "total": len(tests), "error": "timeout"}127 except Exception as e:128 res = {"passed": 0, "total": len(tests), "error": type(e).__name__}129 res["pass_rate"] = res["passed"] / max(1, res["total"])130 res["correct"] = float(res["passed"] == res["total"])131 return res132