1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""End-to-end: run_arm on 5 synthetic properties with the mock provider (no keys, no network)."""3from __future__ import annotations45import json6from pathlib import Path78import polars as pl9import pytest1011from agentic_valuation.config import SYNTHETIC12from agentic_valuation.debate.aggregation import RULES13from agentic_valuation.experiments.freeze import build_freeze, check_freeze, write_freeze14from agentic_valuation.experiments.run_arm import PREDICTION_COLUMNS, RunSpec, prediction_rows, run_arm15from agentic_valuation.schemas import DebateTranscript1617pytestmark = pytest.mark.skipif(not (SYNTHETIC / "synthetic_transactions.parquet").exists() and not (SYNTHETIC / "analytic.parquet").exists(), reason="synthetic data not generated")181920def test_run_arm_end_to_end(tmp_path: Path, monkeypatch):21 monkeypatch.setenv("AV_PROVIDER_MODE", "mock")22 spec = RunSpec(arm="G_debate", rounds=1, committee="mixed_random", split="validation", n=5, synthetic=True, out_dir=tmp_path, run_id="t_g1")23 run_dir = run_arm(spec)24 preds = pl.read_parquet(run_dir / "predictions.parquet")25 assert preds.columns == list(PREDICTION_COLUMNS)26 assert preds.schema["point"] == pl.Float64 and preds.schema["tokens"] == pl.Int6427 assert preds["tx_id"].n_unique() == 5 and preds.height == 5 * (len(RULES) + 1) # + mixture row28 assert set(preds["aggregation"].unique()) == set(RULES) | {"mixture"}29 assert set(preds["arm"].unique()) == {"G_debate_r1"} and set(preds["committee"].unique()) == {"mixed_random"} and set(preds["split"].unique()) == {"validation"}30 assert (preds["q05"] <= preds["q50"]).all() and (preds["q50"] <= preds["q95"]).all() and (preds["point"] > 0).all()31 assert len(list((run_dir / "transcripts").glob("*.json"))) == 5 and len(list((run_dir / "packets").glob("*.json"))) == 532 m = json.loads((run_dir / "manifest.json").read_text())33 assert m["mode"] == "mock" and m["n_transcripts"] == 5 and m["n_failed_this_session"] == 0 and m["arm_label"] == "G_debate_r1"34 assert set(m["prompt_hashes"]) == set(m["roles"]) and len(m["models"]) == 2 and m["config_hashes"] and m["schema_hash"]35 tr = DebateTranscript.model_validate_json(next((run_dir / "transcripts").glob("*.json")).read_text())36 assert tr.rounds <= 1 and len(tr.final_estimates) == 12 and set(tr.assignment.values()) == set(m["models"])37 pk = json.loads(next((run_dir / "packets").glob("*.json")).read_text())38 assert "price" not in pk["subject"] and "street" not in json.dumps(pk).lower()39 # resumable: a second invocation skips everything and rebuilds the same predictions40 run_arm(spec)41 m2 = json.loads((run_dir / "manifest.json").read_text())42 assert m2["n_resumed_skipped"] == 5 and m2["n_transcripts"] == 543 assert pl.read_parquet(run_dir / "predictions.parquet").height == preds.height444546def test_run_arm_spend_cap_graceful(tmp_path: Path, monkeypatch):47 monkeypatch.setenv("AV_PROVIDER_MODE", "mock")48 monkeypatch.setenv("AV_MAX_SPEND_USD", "0.00001")49 spec = RunSpec(arm="F_independent", committee="all_anthropic", split="validation", n=3, synthetic=True, out_dir=tmp_path, run_id="t_cap")50 run_dir = run_arm(spec)51 m = json.loads((run_dir / "manifest.json").read_text())52 assert m["spend_cap_hit"] is True and m["n_transcripts"] == 0 and (run_dir / "predictions.parquet").exists()53 assert pl.read_parquet(run_dir / "predictions.parquet").height == 0545556def test_prediction_rows_contract():57 tr = DebateTranscript(tx_id="x", arm="F_independent", committee="c", protocol="independent", rounds=0, packet_hash="h", turns=[], final_estimates={"a": 100.0, "b": 120.0}, final_quantiles={"a": [80, 90, 100, 110, 130], "b": [95, 110, 120, 130, 150]}, aggregated={"mean": 110.0})58 rows = prediction_rows(tr, "test", "rid")59 assert [r["aggregation"] for r in rows] == ["mean", "mixture"] and set(rows[0]) == set(PREDICTION_COLUMNS)606162def test_freeze_roundtrip(tmp_path: Path):63 p = write_freeze(tmp_path / "freeze.json")64 doc = json.loads(p.read_text())65 assert len(doc["role_prompt_sha256"]) == 21 + 12 and "models.yaml" in doc["configs_sha256"] and "debate/protocols.py" in doc["code_sha256"]66 assert doc["model_ids"]["cost_matched_pairs"]["mid"]["anthropic"] == build_freeze()["model_ids"]["cost_matched_pairs"]["mid"]["anthropic"]67 assert check_freeze(p) == []68