1#!/usr/bin/env python32"""PGAR runtime gate (oracle-free). Standalone: standard library only, so that it runs with any Python ≥ 3.9.34Usage: python pgar_gate.py <workspace_dir> [--python <interpreter>] [--no-repro] [--json]5Reads <workspace_dir>/gate_spec.json and checks the OUTPUT CONTRACT and REPRODUCIBILITY of the deliverables6under <workspace_dir>/out. It contains no reference answers and never judges scientific correctness.7Writes a receipt to <workspace_dir>/out/state/gate_receipts/<utc>.json and exits 0 if passed, 1 otherwise.8"""9import csv, datetime, json, math, os, re, shutil, subprocess, sys, tempfile, hashlib1011GATE_VERSION = "pgar_gate/1.0"12NUMERIC_RE = r"^#+\s*"1314def _utc():15 return datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")1617def _load_json(path):18 try:19 with open(path) as fh:20 return json.load(fh), None21 except FileNotFoundError:22 return None, "missing"23 except Exception as e: # noqa: BLE00124 return None, f"invalid JSON: {e}"2526def _is_num(v):27 return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v)2829def _type_ok(v, t, nullable):30 if v is None:31 return bool(nullable)32 return {"number": _is_num(v), "integer": isinstance(v, int) and not isinstance(v, bool),33 "string": isinstance(v, str), "boolean": isinstance(v, bool), "array": isinstance(v, list),34 "object": isinstance(v, dict)}.get(t, True)3536def _parse_date(s):37 try:38 return datetime.date.fromisoformat(str(s)[:10])39 except Exception: # noqa: BLE00140 return None4142def _compare_results(a, b, rel):43 """Return list of keys whose values differ beyond rel tolerance (numbers) or are unequal (others)."""44 diffs = []45 keys = set(a) | set(b)46 for k in keys:47 va, vb = a.get(k), b.get(k)48 if _is_num(va) and _is_num(vb):49 if abs(va - vb) > rel * max(1.0, abs(va), abs(vb)):50 diffs.append(k)51 elif isinstance(va, (list, dict)) or isinstance(vb, (list, dict)):52 if json.dumps(va, sort_keys=True) != json.dumps(vb, sort_keys=True):53 # allow numeric tolerance inside nested structures54 try:55 fa = [float(x) for x in re.findall(r"-?\d+\.\d+(?:e-?\d+)?", json.dumps(va))]56 fb = [float(x) for x in re.findall(r"-?\d+\.\d+(?:e-?\d+)?", json.dumps(vb))]57 if len(fa) != len(fb) or any(abs(x - y) > rel * max(1.0, abs(x), abs(y)) for x, y in zip(fa, fb)):58 diffs.append(k)59 elif re.sub(r"-?\d+\.\d+(?:e-?\d+)?", "#", json.dumps(va, sort_keys=True)) != re.sub(r"-?\d+\.\d+(?:e-?\d+)?", "#", json.dumps(vb, sort_keys=True)):60 diffs.append(k)61 except Exception: # noqa: BLE00162 diffs.append(k)63 elif va != vb:64 diffs.append(k)65 return sorted(diffs)6667def run_gate(ws, python=None, do_repro=True):68 ws = os.path.abspath(ws)69 checks = []70 def add(cid, ok, detail=""):71 checks.append({"id": cid, "passed": bool(ok), "detail": detail})72 spec, err = _load_json(os.path.join(ws, "gate_spec.json"))73 if spec is None:74 add("gate_spec", False, f"gate_spec.json {err}")75 return _finish(ws, checks)76 # 1. required files77 for rel in spec.get("required_files", []):78 p = os.path.join(ws, rel)79 ok = os.path.isfile(p) and os.path.getsize(p) > 080 add(f"file:{rel}", ok, "present" if ok else "missing or empty")81 # 2. results.json keys/types82 res, err = _load_json(os.path.join(ws, "out", "results.json"))83 if not isinstance(res, dict):84 add("results_json", False, f"out/results.json {err or 'is not an object'}")85 res = {}86 else:87 add("results_json", True, "parsed")88 for k, rule in spec.get("results_keys", {}).items():89 if k not in res:90 add(f"key:{k}", False, "missing")91 else:92 ok = _type_ok(res[k], rule.get("type"), rule.get("nullable", False))93 add(f"key:{k}", ok, "ok" if ok else f"type/nullability mismatch (expected {rule.get('type')}, nullable={rule.get('nullable', False)}, got {type(res[k]).__name__}={res[k]!r:.60})")94 if spec.get("task_id") and res.get("task_id") not in (None, spec["task_id"]):95 add("task_id_matches", False, f"results.task_id={res.get('task_id')!r} != {spec['task_id']!r}")96 # 3. report sections97 rep = ""98 try:99 rep = open(os.path.join(ws, "out", "report.md"), encoding="utf-8", errors="replace").read()100 except Exception: # noqa: BLE001101 pass102 for s in spec.get("report_sections", []):103 ok = re.search(NUMERIC_RE + re.escape(s) + r"\s*$", rep, re.M | re.I) is not None104 add(f"section:{s}", ok, "present" if ok else "heading not found")105 fam = set(spec.get("family_checks", []))106 # 4. claims107 claims, cerr = _load_json(os.path.join(ws, "out", "claims.json"))108 if "claims_reference_existing_keys" in fam or os.path.exists(os.path.join(ws, "out", "claims.json")):109 if not isinstance(claims, list) or not claims:110 add("claims_json", False, f"out/claims.json {cerr or 'must be a non-empty list'}")111 else:112 bad = []113 for i, c in enumerate(claims):114 if not isinstance(c, dict) or not all(k in c for k in ("id", "text", "results_keys", "status")):115 bad.append(f"#{i}: missing fields"); continue116 if c["status"] not in ("supported", "unsupported", "not_evaluated"):117 bad.append(f"{c.get('id')}: bad status {c['status']!r}")118 missing = [k for k in (c.get("results_keys") or []) if k not in res]119 if missing:120 bad.append(f"{c.get('id')}: results_keys not in results.json: {missing}")121 add("claims_json", not bad, "; ".join(bad) if bad else f"{len(claims)} claims, all keys resolve")122 # 5. phase ledger123 pl_path = os.path.join(ws, "out", "state", "phases.json")124 if "phase_ledger_consistent" in fam and os.path.exists(pl_path):125 pl, perr = _load_json(pl_path)126 if not isinstance(pl, dict):127 add("phase_ledger", False, f"phases.json {perr or 'not an object'}")128 else:129 bad = []130 for pid, st in pl.items():131 if not isinstance(st, dict) or st.get("status") not in ("pending", "running", "passed", "failed", "blocked", "stale", "skipped_optional"):132 bad.append(f"{pid}: invalid status"); continue133 if st.get("status") == "passed" and not st.get("checks"):134 bad.append(f"{pid}: passed without recorded checks")135 not_done = [pid for pid, st in pl.items() if isinstance(st, dict) and st.get("status") not in ("passed", "skipped_optional", "blocked")]136 if not_done:137 bad.append(f"phases not terminal: {not_done}")138 add("phase_ledger", not bad, "; ".join(bad) if bad else "consistent")139 # 6. exclusion ledger140 ex_path = os.path.join(ws, "out", "exclusions.csv")141 if "exclusion_ledger_sums" in fam and os.path.exists(ex_path):142 try:143 rows = list(csv.DictReader(open(ex_path, newline="")))144 bad = []145 prev_after = None146 for i, r in enumerate(rows):147 nb, na, nr = int(float(r["n_before"])), int(float(r["n_after"])), int(float(r["n_removed"]))148 if nb - nr != na:149 bad.append(f"row {i}: n_before - n_removed != n_after")150 if prev_after is not None and nb != prev_after:151 bad.append(f"row {i}: n_before != previous n_after")152 prev_after = na153 add("exclusion_ledger", not bad, "; ".join(bad) if bad else f"{len(rows)} steps consistent")154 except Exception as e: # noqa: BLE001155 add("exclusion_ledger", False, f"unreadable: {e}")156 # 7. numeric sanity checks157 if "ci_contains_estimate" in fam:158 for est, lo, hi in spec.get("ci_pairs", []):159 e, l, h = res.get(est), res.get(lo), res.get(hi)160 if all(_is_num(v) for v in (e, l, h)):161 add(f"ci:{est}", l <= e <= h, f"{l} <= {e} <= {h}")162 elif e is None and l is None and h is None:163 add(f"ci:{est}", True, "all null (allowed if reported infeasible)")164 else:165 add(f"ci:{est}", False, "non-numeric CI triple")166 if "positive_se" in fam:167 for k in spec.get("positive_keys", []):168 v = res.get(k)169 add(f"positive:{k}", (v is None) or (_is_num(v) and v > 0), f"{v}")170 if "chronology" in fam:171 for a, b in spec.get("chronology_pairs", []):172 da, db = _parse_date(res.get(a)), _parse_date(res.get(b))173 add(f"chronology:{a}<={b}", da is not None and db is not None and da <= db, f"{res.get(a)} vs {res.get(b)}")174 if "forecasts_csv_chronology" in fam:175 fp = os.path.join(ws, "out", "forecasts.csv")176 try:177 rows = list(csv.DictReader(open(fp, newline="")))178 bad = sum(1 for r in rows if not (_parse_date(r.get("origin_date")) and _parse_date(r.get("target_date")) and _parse_date(r["origin_date"]) < _parse_date(r["target_date"])))179 add("forecasts_chronology", rows and bad == 0, f"{bad} rows with target_date <= origin_date out of {len(rows)}")180 except Exception as e: # noqa: BLE001181 add("forecasts_chronology", False, f"forecasts.csv unreadable: {e}")182 # 8. reproducibility183 if do_repro and os.path.isfile(os.path.join(ws, "out", "analysis.py")) and isinstance(res, dict) and res:184 py = python or sys.executable185 tmp = tempfile.mkdtemp(prefix="pgar_repro_")186 try:187 shutil.copytree(os.path.join(ws, "data"), os.path.join(tmp, "data"))188 shutil.copytree(os.path.join(ws, "out"), os.path.join(tmp, "out"), ignore=shutil.ignore_patterns("results.json", "state", "review"))189 for extra in ("TASK.md", "gate_spec.json"):190 if os.path.exists(os.path.join(ws, extra)):191 shutil.copy(os.path.join(ws, extra), tmp)192 t0 = datetime.datetime.now()193 proc = subprocess.run([py, "out/analysis.py"], cwd=tmp, capture_output=True, text=True, timeout=int(spec.get("analysis_timeout_s", 300)))194 dur = (datetime.datetime.now() - t0).total_seconds()195 new, nerr = _load_json(os.path.join(tmp, "out", "results.json"))196 if proc.returncode != 0:197 add("reproducibility", False, f"analysis.py exit {proc.returncode} after {dur:.1f}s; stderr tail: {proc.stderr[-600:]}")198 elif not isinstance(new, dict):199 add("reproducibility", False, f"analysis.py did not regenerate out/results.json ({nerr})")200 else:201 diffs = _compare_results(res, new, float(spec.get("reproducibility_tolerance_rel", 1e-6)))202 add("reproducibility", not diffs, f"regenerated in {dur:.1f}s; " + ("identical within tolerance" if not diffs else f"differences in keys: {diffs}"))203 except subprocess.TimeoutExpired:204 add("reproducibility", False, "analysis.py timed out")205 except Exception as e: # noqa: BLE001206 add("reproducibility", False, f"repro error: {e}")207 finally:208 shutil.rmtree(tmp, ignore_errors=True)209 elif do_repro:210 add("reproducibility", False, "out/analysis.py or out/results.json missing")211 return _finish(ws, checks)212213def _finish(ws, checks):214 passed = all(c["passed"] for c in checks) and bool(checks)215 receipt = {"gate_version": GATE_VERSION, "workspace": ws, "utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),216 "passed": passed, "n_checks": len(checks), "n_failed": sum(1 for c in checks if not c["passed"]), "checks": checks}217 try:218 d = os.path.join(ws, "out", "state", "gate_receipts"); os.makedirs(d, exist_ok=True)219 path = os.path.join(d, _utc() + ".json")220 with open(path, "w") as fh:221 json.dump(receipt, fh, indent=1)222 receipt["receipt_path"] = path223 receipt["receipt_sha256"] = hashlib.sha256(open(path, "rb").read()).hexdigest()224 except Exception as e: # noqa: BLE001225 receipt["receipt_path"] = None; receipt["receipt_error"] = str(e)226 return receipt227228def main(argv):229 if len(argv) < 2:230 print(__doc__); return 2231 ws = argv[1]; py = None; repro = True; as_json = False232 i = 2233 while i < len(argv):234 if argv[i] == "--python": py = argv[i + 1]; i += 2235 elif argv[i] == "--no-repro": repro = False; i += 1236 elif argv[i] == "--json": as_json = True; i += 1237 else: i += 1238 r = run_gate(ws, py, repro)239 if as_json:240 print(json.dumps(r, indent=1))241 else:242 print(f"PGAR GATE {r['gate_version']} — {'PASSED' if r['passed'] else 'FAILED'} ({r['n_failed']} of {r['n_checks']} checks failed)")243 for c in r["checks"]:244 print(f" [{'ok' if c['passed'] else 'FAIL'}] {c['id']}: {c['detail']}")245 print(f"receipt: {r.get('receipt_path')}")246 return 0 if r["passed"] else 1247248if __name__ == "__main__":249 sys.exit(main(sys.argv))250