1"""Minimal reproduction profile: rerun a fixed small subset from a clean process and compare deterministic outputs2(greedy decoding) with the original records; rebuild analysis tables from raw records without inference."""3from __future__ import annotations4import json, yaml, subprocess, sys, shutil, time5from pathlib import Path6from .util import ROOT, read_jsonl, write_json, utc_now78def main(profile: str) -> int:9 prof = yaml.safe_load(open(profile))10 report = {"started_utc": utc_now(), "profile": prof, "steps": []}11 # 1. inference reproduction: run the profile config into a fresh experiment id, then compare predictions by trial key12 cfg_path = ROOT / prof["config"]; cfg = yaml.safe_load(open(cfg_path))13 orig = {r["trial_id"]: r for r in read_jsonl(ROOT / "runs/raw" / f"{cfg['experiment_id']}.jsonl")} if (ROOT / "runs/raw" / f"{cfg['experiment_id']}.jsonl").exists() else {}14 tmp_cfg = dict(cfg); tmp_cfg["experiment_id"] = cfg["experiment_id"] + "_repro"15 # keep trial keys identical: trial_key uses experiment_id -> compute keys with original id by passing an override16 tmp_cfg["trial_key_experiment_id"] = cfg["experiment_id"]17 # trial keys include the protocol hash: take it from the frozen protocol so keys match the original campaign18 if (ROOT / "protocol/frozen.json").exists():19 tmp_cfg["protocol_hash"] = json.load(open(ROOT / "protocol/frozen.json"))["protocol_hash"]20 tmp_path = ROOT / "reproduction" / "_repro_config.yaml"; yaml.safe_dump(tmp_cfg, open(tmp_path, "w"))21 out = ROOT / "runs/raw" / f"{tmp_cfg['experiment_id']}.jsonl"22 if out.exists(): out.unlink()23 t0 = time.time()24 rc = subprocess.call([sys.executable, "-m", "wp30", "run", "--config", str(tmp_path)], cwd=ROOT)25 rep = {r["trial_id"]: r for r in read_jsonl(out)}26 matched = sum(1 for k, r in rep.items() if k in orig and orig[k]["output_hash"] == r["output_hash"])27 compared = sum(1 for k in rep if k in orig)28 score_match = sum(1 for k, r in rep.items() if k in orig and json.dumps(orig[k]["task_scores"], sort_keys=True) == json.dumps(r["task_scores"], sort_keys=True))29 report["steps"].append({"step": "inference_rerun", "exit_code": rc, "seconds": time.time() - t0, "reran": len(rep), "compared_with_original": compared,30 "identical_output_hash": matched, "identical_task_scores": score_match})31 # 2. analysis-only reproduction32 t0 = time.time()33 rc2 = subprocess.call([sys.executable, "-m", "wp30", "analyze", "--protocol", "protocol/frozen.json"], cwd=ROOT)34 report["steps"].append({"step": "analysis_rebuild", "exit_code": rc2, "seconds": time.time() - t0})35 report["finished_utc"] = utc_now()36 write_json(ROOT / "reproduction/report.json", report)37 print(json.dumps(report["steps"], indent=1))38 return 0 if rc == 0 and rc2 == 0 else 139