"""Collect results/signature_benchmarks/bench_*.json into results/signature_benchmarks/summary.json (headline numbers per host: correctness gates, sign/verify ns per op, scalar-mult, best batch speedups, Python baselines) and print a Markdown table to stdout. Usage: .venv/bin/python experiments/signature_benchmarks/summarize.py """ from __future__ import annotations import datetime as dt, json, sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] RES = ROOT / "results" / "signature_benchmarks" def main(): hosts = {} for f in sorted(RES.glob("bench_*.json")): d = json.loads(f.read_text()) m = d["measurements"] sv = {k: round(v["median_ns"], 1) for k, v in m["sign_verify"].items()} sm = {k: round(v["median_ns"], 1) for k, v in m["scalar_mult"].items() if isinstance(v, dict) and "median_ns" in v} bv = m["batch_verification"] batch = {} for r in bv["results"]: batch.setdefault(r["algorithm"], {})[str(r["batch_size"])] = { "ns_per_sig": round(r["per_signature"]["median_ns"], 1), "iqr_ns": round(r["per_signature"]["iqr_ns"], 1), "msm_only_ns_per_sig": round(r["msm_only_ns_per_sig_median"], 1), "speedup_vs_k256": [round(r["speedup_vs_k256_single_verify"][k], 3) for k in ("point", "ci95_low", "ci95_high")], "speedup_vs_libsecp256k1": [round(r["speedup_vs_libsecp256k1_single_verify"][k], 3) for k in ("point", "ci95_low", "ci95_high")], "window_bits": r["window_bits"], "ops_per_sig": {k: round(v, 1) for k, v in r["op_counts_per_signature"].items()}, } best = {} for alg, per in batch.items(): b, row = max(per.items(), key=lambda kv: kv[1]["speedup_vs_k256"][0]) best[alg] = {"batch_size": int(b), **row} py = m.get("python_baselines") or {} pyr = {k: round(v["median_ns"], 1) for k, v in (py.get("results") or {}).items()} hosts[d["label"]] = { "file": f.name, "hostname": d["host"]["hostname"], "cpu": d["host"]["cpu"], "cores": d["host"]["cores"], "ram_gb": d["host"]["ram_gb"], "os": d["host"]["os"], "timestamp_utc": d["timestamp_utc"], "seed": d["seed"], "rustc": d["toolchain"]["rustc"], "crates": {k: d["toolchain"]["crates"].get(k) for k in ("secp256k1", "secp256k1-sys", "k256")}, "correctness_gates": m["correctness_gates"], "bip340_vectors": {lib: {"verify_pass": m["bip340_vectors"][lib]["verify_pass"], "sign_pass": m["bip340_vectors"][lib]["sign_pass"], "n": m["bip340_vectors"][lib]["n_vectors"]} for lib in ("libsecp256k1", "k256")}, "cross_library_checks": m["cross_library_checks"], "python_reference_crosscheck": {k: v for k, v in m["python_reference_crosscheck"].items() if k != "source"}, "sign_verify_median_ns": sv, "scalar_mult_median_ns": sm, "single_verify_reference_ns": bv["single_verify_reference_ns"], "batch_verification": batch, "best_batch_speedup_per_algorithm": best, "negative_controls": bv["negative_controls"]["all_invalid_batches_rejected"], "python_baselines_median_ns": pyr, "python_versions": py.get("versions"), "rust_elapsed_s": m["rust_elapsed_s"], } summary = {"generated_utc": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), "n_hosts": len(hosts), "hosts": hosts, "classification": { "batch_verification_speedup": "known_reproduced", "implementation_spread": "empirical_observation", "bip340_vectors_and_cross_implementation_agreement": "known_reproduced", }} (RES / "summary.json").write_text(json.dumps(summary, indent=1)) # Markdown digest print("| host | ecdsa sign/verify libsecp | ecdsa sign/verify k256 | schnorr sign/verify libsecp | schnorr sign/verify k256 | best batch (alg, b, ns/sig, S vs k256, S vs libsecp) |") print("|---|---|---|---|---|---|") for lab, h in hosts.items(): s = h["sign_verify_median_ns"] alg, row = max(h["best_batch_speedup_per_algorithm"].items(), key=lambda kv: kv[1]["speedup_vs_k256"][0]) print(f"| {lab} | {s['ecdsa_sign_libsecp256k1']:.0f} / {s['ecdsa_verify_libsecp256k1']:.0f} | {s['ecdsa_sign_k256']:.0f} / {s['ecdsa_verify_k256']:.0f} " f"| {s['schnorr_sign_libsecp256k1']:.0f} / {s['schnorr_verify_libsecp256k1']:.0f} | {s['schnorr_sign_k256']:.0f} / {s['schnorr_verify_k256']:.0f} " f"| {alg}, b={row['batch_size']}, {row['ns_per_sig']:.0f} ns, {row['speedup_vs_k256'][0]:.2f} [{row['speedup_vs_k256'][1]:.2f},{row['speedup_vs_k256'][2]:.2f}], {row['speedup_vs_libsecp256k1'][0]:.2f} |") print(f"\nwrote {RES / 'summary.json'}", file=sys.stderr) if __name__ == "__main__": main()