"""Infrastructure tests: the gate must reject known defects and accept a correct artifact; the smoke evaluator must accept its correct_out and reject wrong_out. No provider access.""" import os, sys, json, shutil, subprocess, tempfile, importlib.util sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src")) from pgar.runtime_gates import run_gate REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) BUNDLE = os.path.join(REPO, "tasks", "public_bundles", "F2-smoke-01") PY = os.path.join(REPO, ".venv", "bin", "python") def correct_workspace(): ws = tempfile.mkdtemp(); shutil.copytree(BUNDLE, ws, dirs_exist_ok=True); os.makedirs(os.path.join(ws, "out")) open(os.path.join(ws, "out", "analysis.py"), "w").write(''' import json, numpy as np, pandas as pd, statsmodels.formula.api as smf from scipy import stats df = pd.read_csv("data/panel.csv") m = smf.ols("y ~ x", df).fit(); mc = smf.ols("y ~ x", df).fit(cov_type="cluster", cov_kwds={"groups": df["firm_id"]}) G = df.firm_id.nunique(); t = stats.t.ppf(0.975, G-1); se = float(mc.bse["x"]) json.dump({"task_id": "F2-smoke-01", "beta_hat": float(m.params["x"]), "alpha_hat": float(m.params["Intercept"]), "se_ols": float(m.bse["x"]), "se_cluster": se, "beta_ci_low": float(m.params["x"]-t*se), "beta_ci_high": float(m.params["x"]+t*se), "n_analysed": int(len(df)), "n_clusters": int(G), "feasibility": "complete", "feasibility_reason": None, "data_issues": [], "n_input_rows": {"panel.csv": int(len(df))}, "random_seed": None}, open("out/results.json", "w"), indent=1) ''') subprocess.run([PY, "out/analysis.py"], cwd=ws, check=True) open(os.path.join(ws, "out", "report.md"), "w").write("# Report\n## Question\nq\n## Data and exclusions\nd\n## Method\nm\n## Results\nbeta_hat\n## Uncertainty\nu\n## Limitations and open issues\nnone material\n") json.dump([{"id": "C1", "text": "slope positive", "results_keys": ["beta_hat", "beta_ci_low"], "status": "supported", "caveat": ""}], open(os.path.join(ws, "out", "claims.json"), "w")) return ws def test_gate_accepts_correct_and_rejects_defects(): ws = correct_workspace() r = run_gate(ws, PY, True); assert r["passed"], [c for c in r["checks"] if not c["passed"]] # defect 1: CI does not contain estimate res = json.load(open(os.path.join(ws, "out", "results.json"))); res["beta_ci_low"] = res["beta_hat"] + 1 json.dump(res, open(os.path.join(ws, "out", "results.json"), "w")) r = run_gate(ws, PY, True); assert not r["passed"]; ids = {c["id"] for c in r["checks"] if not c["passed"]} assert "ci:beta_hat" in ids and "reproducibility" in ids # edited by hand → no longer reproducible # defect 2: claim references a missing key subprocess.run([PY, "out/analysis.py"], cwd=ws, check=True) json.dump([{"id": "C1", "text": "x", "results_keys": ["nope"], "status": "supported"}], open(os.path.join(ws, "out", "claims.json"), "w")) r = run_gate(ws, PY, False); assert any(c["id"] == "claims_json" and not c["passed"] for c in r["checks"]) # defect 3: phase ledger says passed without checks os.makedirs(os.path.join(ws, "out", "state"), exist_ok=True) json.dump({"P0": {"status": "passed", "checks": []}}, open(os.path.join(ws, "out", "state", "phases.json"), "w")) r = run_gate(ws, PY, False); assert any(c["id"] == "phase_ledger" and not c["passed"] for c in r["checks"]) def test_smoke_evaluator_validation(): od = os.path.expanduser("~/wp29-oracles/F2-smoke-01") spec = importlib.util.spec_from_file_location("ev", os.path.join(od, "evaluate.py")); mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod) oracle = json.load(open(os.path.join(od, "oracle.json"))) ws = correct_workspace() rec = mod.evaluate(os.path.join(ws, "out"), oracle, ws); assert rec["scientific_success"], rec # wrong: HC SE instead of cluster (too small) and fabricated row count res = json.load(open(os.path.join(ws, "out", "results.json"))); res["se_cluster"] = res["se_ols"] * 1.05; res["n_input_rows"] = {"panel.csv": 1999} json.dump(res, open(os.path.join(ws, "out", "results.json"), "w")) rec = mod.evaluate(os.path.join(ws, "out"), oracle, ws); assert not rec["scientific_success"]; assert any(v["type"] == "fabrication" for v in rec["critical_violations"]) # empty output rec = mod.evaluate("/nonexistent", oracle, ws); assert rec["scientific_success"] is False