"""Minimal reproduction profile: rerun a fixed small subset from a clean process and compare deterministic outputs (greedy decoding) with the original records; rebuild analysis tables from raw records without inference.""" from __future__ import annotations import json, yaml, subprocess, sys, shutil, time from pathlib import Path from .util import ROOT, read_jsonl, write_json, utc_now def main(profile: str) -> int: prof = yaml.safe_load(open(profile)) report = {"started_utc": utc_now(), "profile": prof, "steps": []} # 1. inference reproduction: run the profile config into a fresh experiment id, then compare predictions by trial key cfg_path = ROOT / prof["config"]; cfg = yaml.safe_load(open(cfg_path)) orig = {r["trial_id"]: r for r in read_jsonl(ROOT / "runs/raw" / f"{cfg['experiment_id']}.jsonl")} if (ROOT / "runs/raw" / f"{cfg['experiment_id']}.jsonl").exists() else {} tmp_cfg = dict(cfg); tmp_cfg["experiment_id"] = cfg["experiment_id"] + "_repro" # keep trial keys identical: trial_key uses experiment_id -> compute keys with original id by passing an override tmp_cfg["trial_key_experiment_id"] = cfg["experiment_id"] # trial keys include the protocol hash: take it from the frozen protocol so keys match the original campaign if (ROOT / "protocol/frozen.json").exists(): tmp_cfg["protocol_hash"] = json.load(open(ROOT / "protocol/frozen.json"))["protocol_hash"] tmp_path = ROOT / "reproduction" / "_repro_config.yaml"; yaml.safe_dump(tmp_cfg, open(tmp_path, "w")) out = ROOT / "runs/raw" / f"{tmp_cfg['experiment_id']}.jsonl" if out.exists(): out.unlink() t0 = time.time() rc = subprocess.call([sys.executable, "-m", "wp30", "run", "--config", str(tmp_path)], cwd=ROOT) rep = {r["trial_id"]: r for r in read_jsonl(out)} matched = sum(1 for k, r in rep.items() if k in orig and orig[k]["output_hash"] == r["output_hash"]) compared = sum(1 for k in rep if k in orig) score_match = sum(1 for k, r in rep.items() if k in orig and json.dumps(orig[k]["task_scores"], sort_keys=True) == json.dumps(r["task_scores"], sort_keys=True)) report["steps"].append({"step": "inference_rerun", "exit_code": rc, "seconds": time.time() - t0, "reran": len(rep), "compared_with_original": compared, "identical_output_hash": matched, "identical_task_scores": score_match}) # 2. analysis-only reproduction t0 = time.time() rc2 = subprocess.call([sys.executable, "-m", "wp30", "analyze", "--protocol", "protocol/frozen.json"], cwd=ROOT) report["steps"].append({"step": "analysis_rebuild", "exit_code": rc2, "seconds": time.time() - t0}) report["finished_utc"] = utc_now() write_json(ROOT / "reproduction/report.json", report) print(json.dumps(report["steps"], indent=1)) return 0 if rc == 0 and rc2 == 0 else 1