1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Run-to-run variation of a hosted-LLM arm: compare a repeated run with the original run on the same4properties (same packets, same protocol, same models; only the providers' sampling differs).56Usage: python3 scripts/36_repeat_variation.py --orig final_G_debate_r2_mixed_random_mid --repeat repeat_G_debate_r2_mixed_random_mid7Output: results/evaluation/repeat_variation.json (+ .csv per property)8"""9from __future__ import annotations1011import argparse12import json13import sys14from pathlib import Path1516import numpy as np17import polars as pl1819sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))20from agentic_valuation.config import PROCESSED, RESULTS # noqa: E402212223def main(argv: list[str] | None = None) -> None:24 ap = argparse.ArgumentParser()25 ap.add_argument("--orig", default="final_G_debate_r2_mixed_random_mid")26 ap.add_argument("--repeat", default="_repeat_G_debate_r2_mixed_random_mid")27 ap.add_argument("--aggregation", default="median")28 a = ap.parse_args(argv)29 o = pl.read_parquet(RESULTS / "runs" / a.orig / "predictions.parquet").filter(pl.col("aggregation") == a.aggregation)30 r = pl.read_parquet(RESULTS / "runs" / a.repeat / "predictions.parquet").filter(pl.col("aggregation") == a.aggregation)31 truth = pl.read_parquet(PROCESSED / "analytic.parquet", columns=["tx_id", "price"])32 j = o.join(r, on="tx_id", suffix="_rep").join(truth, on="tx_id")33 if j.height == 0:34 sys.exit("no common properties")35 d = (j["point"].log() - j["point_rep"].log()).to_numpy()36 e1 = (j["point"].log() - j["price"].log()).abs().to_numpy()37 e2 = (j["point_rep"].log() - j["price"].log()).abs().to_numpy()38 w1 = (j["q95"].log() - j["q05"].log()).to_numpy()39 w2 = (j["q95_rep"].log() - j["q05_rep"].log()).to_numpy()40 out = {41 "orig": a.orig, "repeat": a.repeat, "aggregation": a.aggregation, "n": int(j.height),42 "mean_abs_log_diff_points": float(np.mean(np.abs(d))), "median_abs_log_diff_points": float(np.median(np.abs(d))), "max_abs_log_diff_points": float(np.max(np.abs(d))),43 "sd_log_diff": float(np.std(d, ddof=1)) if j.height > 1 else None,44 "mean_abs_log_error_orig": float(e1.mean()), "mean_abs_log_error_repeat": float(e2.mean()),45 "corr_abs_errors": float(np.corrcoef(e1, e2)[0, 1]) if j.height > 2 else None,46 "mean_log_width90_orig": float(w1.mean()), "mean_log_width90_repeat": float(w2.mean()),47 "share_repeat_inside_orig_90": float(((j["point_rep"] >= j["q05"]) & (j["point_rep"] <= j["q95"])).mean()),48 }49 (RESULTS / "evaluation").mkdir(parents=True, exist_ok=True)50 (RESULTS / "evaluation" / "repeat_variation.json").write_text(json.dumps(out, indent=1))51 j.select(["tx_id", "price", "point", "point_rep", "q05", "q95", "q05_rep", "q95_rep"]).write_csv(RESULTS / "evaluation" / "repeat_variation.csv")52 print(json.dumps(out, indent=1))535455if __name__ == "__main__":56 main()57