1"""Thin wrapper around the `sigbench` Rust binary (WP34 Research Axis B).23Builds the crate (unless --no-build), runs it single-threaded, collects its JSON stdout, runs the4Python cross-check against src/wp34 and the Python baselines, and writes5results/signature_benchmarks/bench_<hostname>.json with the wp34.provenance block6(host, cpu, ram, os, versions, seed, command, timestamp) plus rustc/cargo/crate versions.78Usage (from the repository root):9 PYTHONPATH=src .venv/bin/python experiments/signature_benchmarks/run_bench.py --seed 3410Needs only the standard library + src/wp34 (so it also runs with a bare python3 on cluster nodes).11"""12from __future__ import annotations13import argparse, json, os, platform, re, subprocess, sys, time14from pathlib import Path1516HERE = Path(__file__).resolve().parent17ROOT = HERE.parents[1]18sys.path.insert(0, str(ROOT / "src"))19from wp34 import provenance # noqa: E4022021CRATE = HERE / "sigbench"222324def run(cmd, **kw):25 return subprocess.run(cmd, capture_output=True, text=True, check=True, **kw).stdout262728def crate_versions() -> dict:29 tree = run(["cargo", "tree", "--edges", "normal", "--prefix", "depth"], cwd=CRATE)30 wanted = {"secp256k1", "secp256k1-sys", "k256", "elliptic-curve", "ecdsa", "primeorder", "wnaf", "sha2",31 "rand", "rand_chacha", "serde", "serde_json", "clap", "crypto-bigint"}32 out = {}33 for line in tree.splitlines():34 m = re.match(r"(\d+)(\S+) v(\S+)", line.strip())35 if m and m.group(2) in wanted and m.group(2) not in out:36 out[m.group(2)] = m.group(3)37 return out383940def main():41 ap = argparse.ArgumentParser()42 ap.add_argument("--seed", type=int, default=34)43 ap.add_argument("--reps", type=int, default=7)44 ap.add_argument("--batch-ops", type=int, default=2000)45 ap.add_argument("--batch-min-sigs", type=int, default=2000)46 ap.add_argument("--batch-sizes", default="1,2,4,8,16,32,64,128,256,512,1024,2048")47 ap.add_argument("--crosscheck-n", type=int, default=1000)48 ap.add_argument("--python-crosscheck-n", type=int, default=200)49 ap.add_argument("--bootstrap", type=int, default=2000)50 ap.add_argument("--label", default=None, help="human label for figures (e.g. 'Laptop M5 Max')")51 ap.add_argument("--out", default=None, help="default results/signature_benchmarks/bench_<hostname>.json")52 ap.add_argument("--no-build", action="store_true")53 ap.add_argument("--no-python-baselines", action="store_true")54 ap.add_argument("--python-ops", type=int, default=20)55 a = ap.parse_args()56 started = time.time()5758 host = platform.node().split(".")[0]59 out = Path(a.out) if a.out else ROOT / "results" / "signature_benchmarks" / f"bench_{host}.json"60 out.parent.mkdir(parents=True, exist_ok=True)61 cc_path = out.parent / f"crosscheck_triples_{host}.json"6263 if not a.no_build:64 subprocess.run(["cargo", "build", "--release"], cwd=CRATE, check=True)65 binary = CRATE / "target" / "release" / "sigbench"66 vectors = ROOT / "data" / "vectors" / "bip340_test_vectors.csv"67 cmd = [str(binary), "--seed", str(a.seed), "--reps", str(a.reps), "--batch-ops", str(a.batch_ops),68 "--batch-min-sigs", str(a.batch_min_sigs), "--batch-sizes", a.batch_sizes,69 "--crosscheck-n", str(a.crosscheck_n), "--python-crosscheck-n", str(a.python_crosscheck_n),70 "--bootstrap", str(a.bootstrap), "--vectors", str(vectors), "--crosscheck-out", str(cc_path)]71 print("[run_bench] " + " ".join(cmd), file=sys.stderr)72 t0 = time.time()73 proc = subprocess.run(cmd, capture_output=True, text=True, check=True)74 sys.stderr.write(proc.stderr)75 rust = json.loads(proc.stdout)76 rust_elapsed = time.time() - t07778 # Python cross-check of the Rust triples against src/wp34 (exact-arithmetic reference)79 from python_crosscheck import crosscheck80 py_cc = crosscheck(cc_path, resign=True)81 print(f"[run_bench] python crosscheck all_pass={py_cc['all_pass']} n={py_cc['n']} ({py_cc['elapsed_s']} s)", file=sys.stderr)8283 baselines = None84 if not a.no_python_baselines:85 try:86 bl = run([sys.executable, str(HERE / "python_baselines.py"), "--seed", str(a.seed), "--ops", str(a.python_ops),87 "--reps", str(a.reps)], env={**os.environ, "PYTHONPATH": str(ROOT / "src")})88 baselines = json.loads(bl)89 for k, v in baselines["results"].items():90 print(f"[python-baseline] {k:<48} {v['median_ns']:>14.1f} ns/op", file=sys.stderr)91 except subprocess.CalledProcessError as e:92 baselines = {"error": e.stderr[-2000:]}9394 gates = {95 "bip340_vectors_both_libraries": rust["bip340_vectors"]["both_all_pass"],96 "cross_library_1000": rust["cross_library_checks"]["all_pass"],97 "python_reference_200": py_cc["all_pass"],98 "own_scalar_mult_agrees_with_k256": rust["scalar_mult"]["own_wnaf5_agrees_with_k256_64_cases"],99 "batch_negative_controls_rejected": rust["batch_verification"]["negative_controls"]["all_invalid_batches_rejected"],100 }101 gates["all"] = all(gates.values())102 print(f"[run_bench] correctness gates: {gates}", file=sys.stderr)103104 toolchain = {105 "rustc": run(["rustc", "--version"]).strip(),106 "cargo": run(["cargo", "--version"]).strip(),107 "crates": crate_versions(),108 "profile": "release (opt-level 3, lto fat, codegen-units 1)",109 "target": run(["rustc", "-vV"]).split("host:")[1].split()[0],110 }111 params = {"seed": a.seed, "reps": a.reps, "batch_ops": a.batch_ops, "batch_min_sigs": a.batch_min_sigs,112 "batch_sizes": [int(x) for x in a.batch_sizes.split(",")], "crosscheck_n": a.crosscheck_n,113 "python_crosscheck_n": a.python_crosscheck_n, "bootstrap": a.bootstrap, "python_ops": a.python_ops,114 "rust_command": " ".join(cmd)}115 measurements = {116 "correctness_gates": gates,117 "bip340_vectors": rust["bip340_vectors"],118 "cross_library_checks": rust["cross_library_checks"],119 "python_reference_crosscheck": py_cc,120 "sign_verify": rust["sign_verify"],121 "scalar_mult": rust["scalar_mult"],122 "batch_verification": rust["batch_verification"],123 "python_baselines": baselines,124 "rust_elapsed_s": round(rust_elapsed, 2),125 "threading": rust["threading"],126 }127 doc = provenance.record("signature_benchmarks", params, measurements, seed=a.seed,128 extra={"label": a.label or host, "toolchain": toolchain, "crate": "experiments/signature_benchmarks/sigbench"},129 out=out, started=started)130 print(f"[run_bench] wrote {out} ({doc['elapsed_s']} s total)", file=sys.stderr)131132133if __name__ == "__main__":134 main()135