"""Export the validated dataset: one row per run (metrics + receipt + judgments), Parquet + CSV, plus judgments and events tables.""" from __future__ import annotations import json, os import pandas as pd from .config import VALIDATED def export(store, tag="pilot"): os.makedirs(VALIDATED, exist_ok=True) rows = [] interventions = {} for i in store.q("SELECT run_id, SUM(minutes) AS m FROM interventions GROUP BY run_id"): interventions[i["run_id"]] = i["m"] for r in store.q("SELECT m.metrics, r.stage, r.seed FROM metrics m JOIN runs r USING(run_id) WHERE r.stage=?", (tag,)): m = json.loads(r["metrics"]); m["stage"] = r["stage"]; m["seed"] = r["seed"] m["human_intervention_min"] = interventions.get(m["run_id"], 0.0) for k in ("by_tool", "by_family", "family_sequence", "tokens", "model_usage", "quality_scores", "judge_acceptable_votes"): m[k] = json.dumps(m.get(k)) rows.append(m) df = pd.DataFrame(rows) if len(df): df.to_parquet(os.path.join(VALIDATED, f"runs_{tag}.parquet"), index=False); df.to_csv(os.path.join(VALIDATED, f"runs_{tag}.csv"), index=False) j = pd.DataFrame(store.q("SELECT j.run_id, judge_idx, judge_model, score, scores, j.cost_usd, judged_utc FROM judgments j JOIN runs r USING(run_id) WHERE r.stage=?", (tag,))) if len(j): j.to_parquet(os.path.join(VALIDATED, f"judgments_{tag}.parquet"), index=False); j.to_csv(os.path.join(VALIDATED, f"judgments_{tag}.csv"), index=False) rc = pd.DataFrame([{"run_id": x["run_id"], **{k: v for k, v in json.loads(x["receipt"]).items() if k in ("det_completion", "det_score", "evaluator_version", "oracle_sha256")}, "criteria": json.dumps(json.loads(x["receipt"]).get("required_criteria")), "violations": json.dumps(json.loads(x["receipt"]).get("critical_violations"))} for x in store.q("SELECT c.run_id, receipt FROM receipts c JOIN runs r USING(run_id) WHERE r.stage=?", (tag,))]) if len(rc): rc.to_parquet(os.path.join(VALIDATED, f"receipts_{tag}.parquet"), index=False); rc.to_csv(os.path.join(VALIDATED, f"receipts_{tag}.csv"), index=False) e = pd.DataFrame(store.q("SELECT e.run_id, e.seq, e.elapsed_s, e.event_type, e.tool_name, e.status AS status, e.error_class, substr(e.summary,1,300) AS summary FROM events e JOIN runs r USING(run_id) WHERE r.stage=?", (tag,))) if len(e): e.to_parquet(os.path.join(VALIDATED, f"events_{tag}.parquet"), index=False) return len(df), len(j), len(e)