"""Thin wrapper around the `sigbench` Rust binary (WP34 Research Axis B). Builds the crate (unless --no-build), runs it single-threaded, collects its JSON stdout, runs the Python cross-check against src/wp34 and the Python baselines, and writes results/signature_benchmarks/bench_.json with the wp34.provenance block (host, cpu, ram, os, versions, seed, command, timestamp) plus rustc/cargo/crate versions. Usage (from the repository root): PYTHONPATH=src .venv/bin/python experiments/signature_benchmarks/run_bench.py --seed 34 Needs only the standard library + src/wp34 (so it also runs with a bare python3 on cluster nodes). """ from __future__ import annotations import argparse, json, os, platform, re, subprocess, sys, time from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] sys.path.insert(0, str(ROOT / "src")) from wp34 import provenance # noqa: E402 CRATE = HERE / "sigbench" def run(cmd, **kw): return subprocess.run(cmd, capture_output=True, text=True, check=True, **kw).stdout def crate_versions() -> dict: tree = run(["cargo", "tree", "--edges", "normal", "--prefix", "depth"], cwd=CRATE) wanted = {"secp256k1", "secp256k1-sys", "k256", "elliptic-curve", "ecdsa", "primeorder", "wnaf", "sha2", "rand", "rand_chacha", "serde", "serde_json", "clap", "crypto-bigint"} out = {} for line in tree.splitlines(): m = re.match(r"(\d+)(\S+) v(\S+)", line.strip()) if m and m.group(2) in wanted and m.group(2) not in out: out[m.group(2)] = m.group(3) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("--seed", type=int, default=34) ap.add_argument("--reps", type=int, default=7) ap.add_argument("--batch-ops", type=int, default=2000) ap.add_argument("--batch-min-sigs", type=int, default=2000) ap.add_argument("--batch-sizes", default="1,2,4,8,16,32,64,128,256,512,1024,2048") ap.add_argument("--crosscheck-n", type=int, default=1000) ap.add_argument("--python-crosscheck-n", type=int, default=200) ap.add_argument("--bootstrap", type=int, default=2000) ap.add_argument("--label", default=None, help="human label for figures (e.g. 'Laptop M5 Max')") ap.add_argument("--out", default=None, help="default results/signature_benchmarks/bench_.json") ap.add_argument("--no-build", action="store_true") ap.add_argument("--no-python-baselines", action="store_true") ap.add_argument("--python-ops", type=int, default=20) a = ap.parse_args() started = time.time() host = platform.node().split(".")[0] out = Path(a.out) if a.out else ROOT / "results" / "signature_benchmarks" / f"bench_{host}.json" out.parent.mkdir(parents=True, exist_ok=True) cc_path = out.parent / f"crosscheck_triples_{host}.json" if not a.no_build: subprocess.run(["cargo", "build", "--release"], cwd=CRATE, check=True) binary = CRATE / "target" / "release" / "sigbench" vectors = ROOT / "data" / "vectors" / "bip340_test_vectors.csv" cmd = [str(binary), "--seed", str(a.seed), "--reps", str(a.reps), "--batch-ops", str(a.batch_ops), "--batch-min-sigs", str(a.batch_min_sigs), "--batch-sizes", a.batch_sizes, "--crosscheck-n", str(a.crosscheck_n), "--python-crosscheck-n", str(a.python_crosscheck_n), "--bootstrap", str(a.bootstrap), "--vectors", str(vectors), "--crosscheck-out", str(cc_path)] print("[run_bench] " + " ".join(cmd), file=sys.stderr) t0 = time.time() proc = subprocess.run(cmd, capture_output=True, text=True, check=True) sys.stderr.write(proc.stderr) rust = json.loads(proc.stdout) rust_elapsed = time.time() - t0 # Python cross-check of the Rust triples against src/wp34 (exact-arithmetic reference) from python_crosscheck import crosscheck py_cc = crosscheck(cc_path, resign=True) print(f"[run_bench] python crosscheck all_pass={py_cc['all_pass']} n={py_cc['n']} ({py_cc['elapsed_s']} s)", file=sys.stderr) baselines = None if not a.no_python_baselines: try: bl = run([sys.executable, str(HERE / "python_baselines.py"), "--seed", str(a.seed), "--ops", str(a.python_ops), "--reps", str(a.reps)], env={**os.environ, "PYTHONPATH": str(ROOT / "src")}) baselines = json.loads(bl) for k, v in baselines["results"].items(): print(f"[python-baseline] {k:<48} {v['median_ns']:>14.1f} ns/op", file=sys.stderr) except subprocess.CalledProcessError as e: baselines = {"error": e.stderr[-2000:]} gates = { "bip340_vectors_both_libraries": rust["bip340_vectors"]["both_all_pass"], "cross_library_1000": rust["cross_library_checks"]["all_pass"], "python_reference_200": py_cc["all_pass"], "own_scalar_mult_agrees_with_k256": rust["scalar_mult"]["own_wnaf5_agrees_with_k256_64_cases"], "batch_negative_controls_rejected": rust["batch_verification"]["negative_controls"]["all_invalid_batches_rejected"], } gates["all"] = all(gates.values()) print(f"[run_bench] correctness gates: {gates}", file=sys.stderr) toolchain = { "rustc": run(["rustc", "--version"]).strip(), "cargo": run(["cargo", "--version"]).strip(), "crates": crate_versions(), "profile": "release (opt-level 3, lto fat, codegen-units 1)", "target": run(["rustc", "-vV"]).split("host:")[1].split()[0], } params = {"seed": a.seed, "reps": a.reps, "batch_ops": a.batch_ops, "batch_min_sigs": a.batch_min_sigs, "batch_sizes": [int(x) for x in a.batch_sizes.split(",")], "crosscheck_n": a.crosscheck_n, "python_crosscheck_n": a.python_crosscheck_n, "bootstrap": a.bootstrap, "python_ops": a.python_ops, "rust_command": " ".join(cmd)} measurements = { "correctness_gates": gates, "bip340_vectors": rust["bip340_vectors"], "cross_library_checks": rust["cross_library_checks"], "python_reference_crosscheck": py_cc, "sign_verify": rust["sign_verify"], "scalar_mult": rust["scalar_mult"], "batch_verification": rust["batch_verification"], "python_baselines": baselines, "rust_elapsed_s": round(rust_elapsed, 2), "threading": rust["threading"], } doc = provenance.record("signature_benchmarks", params, measurements, seed=a.seed, extra={"label": a.label or host, "toolchain": toolchain, "crate": "experiments/signature_benchmarks/sigbench"}, out=out, started=started) print(f"[run_bench] wrote {out} ({doc['elapsed_s']} s total)", file=sys.stderr) if __name__ == "__main__": main()