"""Executable task family: small, deterministic coding/data tasks whose correctness is checked by hidden tests. Family 'exec': an agent receives (a) a source document describing a small dataset + a bug report or analysis request and (b) a task; it must output a Python function body. Hidden tests (evaluator-only) run in a clean subprocess with a timeout. The 'document' is the information the sender holds (table schema, bug report, expected behaviour); the receiver must reconstruct the specification from the message. Ground truth is generated deterministically from a seed. """ from __future__ import annotations import random, json, textwrap, subprocess, sys, tempfile, os from dataclasses import dataclass, asdict COLS = ["region", "product", "units", "unit_price", "discount_pct", "returned", "shipped_days"] REGIONS = ["Nord", "Sud", "Est", "Ouest", "Centre"] PRODUCTS = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"] def make_table(rng: random.Random, n: int): rows = [] for _ in range(n): rows.append({"region": rng.choice(REGIONS), "product": rng.choice(PRODUCTS), "units": rng.randint(1, 40), "unit_price": round(rng.uniform(2.5, 80.0), 2), "discount_pct": rng.choice([0, 0, 5, 10, 15, 25]), "returned": rng.random() < 0.12, "shipped_days": rng.randint(1, 14)}) return rows def spec_variants(rng: random.Random, rows): """Return (spec_text, reference_fn_source, test_cases). Spec encodes exact business rules that must be transmitted.""" thr_days = rng.randint(3, 9) min_units = rng.randint(2, 6) excl_region = rng.choice(REGIONS) rounding = rng.choice([0, 1, 2]) tax = rng.choice([0.0, 5.0, 9.975, 14.975]) spec = (f"Bug report BR-{rng.randint(1000,9999)}: the function net_revenue(rows) must compute the total net revenue of the " f"rows list (each row is a dict with keys {COLS}). Rules agreed with the client on {rng.randint(1,28)} {rng.choice(['March','June','September'])} 2031: " f"(1) exclude returned rows; (2) exclude rows whose region is '{excl_region}'; (3) exclude rows with fewer than {min_units} units; " f"(4) rows shipped in more than {thr_days} days receive an additional 5 percentage points of discount on top of discount_pct, capped at 50 %; " f"(5) line revenue = units * unit_price * (1 - total_discount/100); (6) apply a tax of {tax} % on the sum; " f"(7) return the result rounded to {rounding} decimals (Python round). The current implementation forgets rule (4) and rule (2).") ref = textwrap.dedent(f""" def net_revenue(rows): total = 0.0 for r in rows: if r['returned']: continue if r['region'] == '{excl_region}': continue if r['units'] < {min_units}: continue d = r['discount_pct'] + (5 if r['shipped_days'] > {thr_days} else 0) d = min(d, 50) total += r['units'] * r['unit_price'] * (1 - d / 100.0) total = total * (1 + {tax} / 100.0) return round(total, {rounding}) """) ns = {} exec(ref, ns) tests = [] for k in range(5): sub = [dict(r) for r in rng.sample(rows, k=min(len(rows), rng.randint(4, len(rows))))] tests.append({"input": sub, "expected": ns["net_revenue"](sub)}) return spec, ref, tests, {"thr_days": thr_days, "min_units": min_units, "excl_region": excl_region, "rounding": rounding, "tax": tax} def generate_exec_task(seed: int, target_tokens: int = 2000) -> dict: rng = random.Random(10_000 + seed) rows = make_table(rng, rng.randint(12, 24)) spec, ref, tests, params = spec_variants(rng, rows) sample_rows = json.dumps(rows[:6], indent=None) filler = [] boiler = ["The ticketing system assigns severity on a four-level scale; this ticket is severity 2.", "All monetary values are in the client's reporting currency; no FX conversion is applied here.", "Unit tests live in the tests/ directory and are executed in CI on every push.", "The data engineering team refreshes the sales extract nightly at 02:00 local time.", "Code style follows the repository's formatter settings; do not reformat unrelated lines.", "Deployment to production requires a reviewer approval and a green pipeline.", "Historic tickets BR-1183 and BR-2047 are unrelated and remain closed.", "The data dictionary defines shipped_days as the number of calendar days from order to shipment."] n_fill = max(0, int(target_tokens * 4 / 95) - 12) filler = [rng.choice(boiler) for _ in range(n_fill)] doc = ("ENGINEERING TICKET\n\n" + spec + "\n\nSample rows (first 6 of the extract):\n" + sample_rows + "\n\n" + "\n".join(filler[: n_fill // 2]) + "\n\nThe existing (buggy) implementation is:\n\n" + "def net_revenue(rows):\n total = 0.0\n for r in rows:\n if r['returned']:\n continue\n" f" if r['units'] < {params['min_units']}:\n continue\n d = r['discount_pct']\n" " total += r['units'] * r['unit_price'] * (1 - d / 100.0)\n" f" total = total * (1 + {params['tax']} / 100.0)\n return round(total, {params['rounding']})\n\n" + "\n".join(filler[n_fill // 2:])) task = ("Rewrite the Python function net_revenue(rows) so that it implements ALL the agreed rules in the ticket. " "Output only Python code defining net_revenue; no explanations, no markdown fences.") return {"document_id": f"exec_{seed:05d}_{target_tokens}", "seed": seed, "target_tokens": target_tokens, "text": doc, "char_len": len(doc), "questions": [{"question_id": "q00_exec", "qtype": "exec", "question": task, "answer": ref, "answer_type": "exec", "tolerance": 0.0, "supporting_fact_ids": ["spec"], "answer_position_frac": 0.02, "aliases": []}], "facts": [{"fact_id": "spec", "entity": "net_revenue", "attribute": "rules", "value": json.dumps(params), "kind": "spec", "sentence": spec, "char_start": doc.find(spec), "char_end": doc.find(spec) + len(spec)}], "hidden_tests": tests} def run_hidden_tests(code: str, tests: list, timeout_s: float = 10.0) -> dict: """Execute candidate code in a clean subprocess (no network, no project imports) against hidden tests.""" code = code.strip() if code.startswith("```"): code = code.strip("`") if code.startswith("python"): code = code[6:] harness = textwrap.dedent(""" import json, sys, math src = open(sys.argv[1]).read(); tests = json.load(open(sys.argv[2])) ns = {} try: exec(src, ns) except Exception as e: print(json.dumps({"passed": 0, "total": len(tests), "error": "exec:" + type(e).__name__})); sys.exit(0) fn = ns.get('net_revenue') if fn is None: print(json.dumps({"passed": 0, "total": len(tests), "error": "no_function"})); sys.exit(0) p = 0; err = None for t in tests: try: out = fn([dict(r) for r in t['input']]) if isinstance(out, (int, float)) and math.isclose(float(out), float(t['expected']), rel_tol=1e-9, abs_tol=1e-6): p += 1 except Exception as e: err = type(e).__name__ print(json.dumps({"passed": p, "total": len(tests), "error": err})) """) with tempfile.TemporaryDirectory() as d: cp = os.path.join(d, "cand.py"); tp = os.path.join(d, "tests.json"); hp = os.path.join(d, "h.py") open(cp, "w").write(code); json.dump(tests, open(tp, "w")); open(hp, "w").write(harness) try: r = subprocess.run([sys.executable, "-I", hp, cp, tp], capture_output=True, text=True, timeout=timeout_s, cwd=d) res = json.loads(r.stdout.strip().splitlines()[-1]) if r.stdout.strip() else {"passed": 0, "total": len(tests), "error": "no_output"} except subprocess.TimeoutExpired: res = {"passed": 0, "total": len(tests), "error": "timeout"} except Exception as e: res = {"passed": 0, "total": len(tests), "error": type(e).__name__} res["pass_rate"] = res["passed"] / max(1, res["total"]) res["correct"] = float(res["passed"] == res["total"]) return res