1"""Collect results/signature_benchmarks/bench_*.json into results/signature_benchmarks/summary.json2(headline numbers per host: correctness gates, sign/verify ns per op, scalar-mult, best batch speedups,3Python baselines) and print a Markdown table to stdout.45Usage: .venv/bin/python experiments/signature_benchmarks/summarize.py6"""7from __future__ import annotations8import datetime as dt, json, sys9from pathlib import Path1011ROOT = Path(__file__).resolve().parents[2]12RES = ROOT / "results" / "signature_benchmarks"131415def main():16 hosts = {}17 for f in sorted(RES.glob("bench_*.json")):18 d = json.loads(f.read_text())19 m = d["measurements"]20 sv = {k: round(v["median_ns"], 1) for k, v in m["sign_verify"].items()}21 sm = {k: round(v["median_ns"], 1) for k, v in m["scalar_mult"].items() if isinstance(v, dict) and "median_ns" in v}22 bv = m["batch_verification"]23 batch = {}24 for r in bv["results"]:25 batch.setdefault(r["algorithm"], {})[str(r["batch_size"])] = {26 "ns_per_sig": round(r["per_signature"]["median_ns"], 1),27 "iqr_ns": round(r["per_signature"]["iqr_ns"], 1),28 "msm_only_ns_per_sig": round(r["msm_only_ns_per_sig_median"], 1),29 "speedup_vs_k256": [round(r["speedup_vs_k256_single_verify"][k], 3) for k in ("point", "ci95_low", "ci95_high")],30 "speedup_vs_libsecp256k1": [round(r["speedup_vs_libsecp256k1_single_verify"][k], 3) for k in ("point", "ci95_low", "ci95_high")],31 "window_bits": r["window_bits"],32 "ops_per_sig": {k: round(v, 1) for k, v in r["op_counts_per_signature"].items()},33 }34 best = {}35 for alg, per in batch.items():36 b, row = max(per.items(), key=lambda kv: kv[1]["speedup_vs_k256"][0])37 best[alg] = {"batch_size": int(b), **row}38 py = m.get("python_baselines") or {}39 pyr = {k: round(v["median_ns"], 1) for k, v in (py.get("results") or {}).items()}40 hosts[d["label"]] = {41 "file": f.name, "hostname": d["host"]["hostname"], "cpu": d["host"]["cpu"], "cores": d["host"]["cores"],42 "ram_gb": d["host"]["ram_gb"], "os": d["host"]["os"], "timestamp_utc": d["timestamp_utc"], "seed": d["seed"],43 "rustc": d["toolchain"]["rustc"], "crates": {k: d["toolchain"]["crates"].get(k) for k in ("secp256k1", "secp256k1-sys", "k256")},44 "correctness_gates": m["correctness_gates"],45 "bip340_vectors": {lib: {"verify_pass": m["bip340_vectors"][lib]["verify_pass"], "sign_pass": m["bip340_vectors"][lib]["sign_pass"],46 "n": m["bip340_vectors"][lib]["n_vectors"]} for lib in ("libsecp256k1", "k256")},47 "cross_library_checks": m["cross_library_checks"],48 "python_reference_crosscheck": {k: v for k, v in m["python_reference_crosscheck"].items() if k != "source"},49 "sign_verify_median_ns": sv,50 "scalar_mult_median_ns": sm,51 "single_verify_reference_ns": bv["single_verify_reference_ns"],52 "batch_verification": batch,53 "best_batch_speedup_per_algorithm": best,54 "negative_controls": bv["negative_controls"]["all_invalid_batches_rejected"],55 "python_baselines_median_ns": pyr,56 "python_versions": py.get("versions"),57 "rust_elapsed_s": m["rust_elapsed_s"],58 }59 summary = {"generated_utc": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),60 "n_hosts": len(hosts), "hosts": hosts,61 "classification": {62 "batch_verification_speedup": "known_reproduced",63 "implementation_spread": "empirical_observation",64 "bip340_vectors_and_cross_implementation_agreement": "known_reproduced",65 }}66 (RES / "summary.json").write_text(json.dumps(summary, indent=1))67 # Markdown digest68 print("| host | ecdsa sign/verify libsecp | ecdsa sign/verify k256 | schnorr sign/verify libsecp | schnorr sign/verify k256 | best batch (alg, b, ns/sig, S vs k256, S vs libsecp) |")69 print("|---|---|---|---|---|---|")70 for lab, h in hosts.items():71 s = h["sign_verify_median_ns"]72 alg, row = max(h["best_batch_speedup_per_algorithm"].items(), key=lambda kv: kv[1]["speedup_vs_k256"][0])73 print(f"| {lab} | {s['ecdsa_sign_libsecp256k1']:.0f} / {s['ecdsa_verify_libsecp256k1']:.0f} | {s['ecdsa_sign_k256']:.0f} / {s['ecdsa_verify_k256']:.0f} "74 f"| {s['schnorr_sign_libsecp256k1']:.0f} / {s['schnorr_verify_libsecp256k1']:.0f} | {s['schnorr_sign_k256']:.0f} / {s['schnorr_verify_k256']:.0f} "75 f"| {alg}, b={row['batch_size']}, {row['ns_per_sig']:.0f} ns, {row['speedup_vs_k256'][0]:.2f} [{row['speedup_vs_k256'][1]:.2f},{row['speedup_vs_k256'][2]:.2f}], {row['speedup_vs_libsecp256k1'][0]:.2f} |")76 print(f"\nwrote {RES / 'summary.json'}", file=sys.stderr)777879if __name__ == "__main__":80 main()81