1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Phase 5 — estimate the aggregation weights (inverse-variance, stacking, robust-Bayes scale) on the VALIDATION4transcripts of the reference debate run, and the conformal scales of the LLM arms' median-of-quantiles intervals.5Both are frozen inputs of the test evaluation (never re-estimated on test).67Usage: python3 scripts/34_fit_weights.py [--run validation_G_debate_r2_mixed_random_mid]8Outputs: results/aggregation_weights.json, results/evaluation/llm_conformal_scales.json9"""10from __future__ import annotations1112import argparse13import json14import sys15from pathlib import Path1617import pandas as pd18import polars as pl1920sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))21from agentic_valuation.config import PROCESSED, RESULTS # noqa: E40222from agentic_valuation.debate.aggregation import fit_weights, save_weights # noqa: E40223from agentic_valuation.evaluation.calibration import conformalise # noqa: E40224from agentic_valuation.schemas import DebateTranscript # noqa: E402252627def load_transcripts(run_dir: Path) -> list[DebateTranscript]:28 return [DebateTranscript.model_validate_json(p.read_text()) for p in sorted((run_dir / "transcripts").glob("*.json"))]293031def main(argv: list[str] | None = None) -> None:32 ap = argparse.ArgumentParser()33 ap.add_argument("--run", default="validation_G_debate_r2_mixed_random_mid")34 a = ap.parse_args(argv)35 truth_df = pl.read_parquet(PROCESSED / "analytic.parquet", columns=["tx_id", "price", "mun_code", "date"])36 truth = dict(zip(truth_df["tx_id"], truth_df["price"]))37 run_dir = RESULTS / "runs" / a.run38 trs = load_transcripts(run_dir)39 w = fit_weights(trs, truth)40 w["source_run"] = a.run41 path = save_weights(w)42 print(f"weights from {len(trs)} validation transcripts -> {path}")43 print({k: round(v, 3) for k, v in w["inverse_variance"].items()})44 print("stacking", {k: round(v, 3) for k, v in w["stacking"].items() if v > 0.001})45 # conformal scales per LLM arm (validation runs only, median aggregation / median_of_quantiles interval)46 scales: dict[str, dict] = {}47 tp = truth_df.to_pandas().rename(columns={"price": "price"})48 for d in sorted((RESULTS / "runs").glob("validation_*")):49 f = d / "predictions.parquet"50 if not f.exists():51 continue52 p = pl.read_parquet(f).filter(pl.col("aggregation") == "median").to_pandas()53 if p.empty or p["q05"].isna().all():54 continue55 try:56 cs = conformalise(p, tp)57 scales[d.name] = {"arm": str(p["arm"].iloc[0]), "committee": str(p["committee"].iloc[0]), "n": int(len(p)), "scales": cs.__dict__ if hasattr(cs, "__dict__") else str(cs)}58 except Exception as e: # noqa: BLE00159 scales[d.name] = {"error": str(e)[:200]}60 out = RESULTS / "evaluation" / "llm_conformal_scales.json"61 out.parent.mkdir(parents=True, exist_ok=True)62 out.write_text(json.dumps(scales, indent=1, default=float))63 print(f"conformal scales for {len(scales)} validation runs -> {out}")646566if __name__ == "__main__":67 main()68