1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2from __future__ import annotations34import pytest56from agentic_valuation.agents.roles import get_role, placebo_roles, roles_for7from agentic_valuation.agents.runner import run_turn8from agentic_valuation.debate.aggregation import RULES9from agentic_valuation.debate.committees import assign10from agentic_valuation.debate.protocols import cost_matched_single, independent, structured_debate11from agentic_valuation.debate.stopping import iqr_log_points, should_stop12from agentic_valuation.providers.ledger import SpendCapExceeded13from agentic_valuation.providers.mock import MockProvider14from agentic_valuation.schemas import AgentEstimate15from tests.conftest_agents import packet, pool # noqa: F4011617ROLES12 = roles_for("default12")18IDS = [r.id for r in ROLES12]192021async def test_independent_12_estimates(packet, pool):22 asg = assign(IDS, "mixed_random", tx_id=packet.subject.tx_id)23 tr = await independent(packet, ROLES12, asg, pool, committee="mixed_random")24 assert len(tr.turns) == 12 and len(tr.final_estimates) == 12 and tr.rounds == 025 assert all(t.kind == "estimate" and t.round == 0 and t.error is None and t.estimate is not None for t in tr.turns)26 assert all(t.input_tokens > 0 and t.prompt_hash and t.model == asg[t.role] for t in tr.turns)27 assert set(RULES) <= set(tr.aggregated) and set(tr.aggregated_interval) == {"median_of_quantiles", "mixture"}28 assert tr.assignment == asg and tr.packet_hash == packet.content_hash()29 assert tr.total_tokens == sum(t.input_tokens + t.output_tokens for t in tr.turns) and tr.total_cost_usd > 030 # single generalist = arm E31 te = await independent(packet, [get_role("generalist")], {"generalist": "claude-sonnet-5-5"}, pool, arm="E_single_llm")32 assert list(te.final_estimates) == ["generalist"] and te.aggregated["median"] == pytest.approx(te.final_estimates["generalist"])333435async def test_structured_debate_two_rounds(packet, pool):36 asg = assign(IDS, "all_anthropic")37 tr = await structured_debate(packet, ROLES12, asg, pool, rounds=2, iqr_tol=0.0) # tol 0 → never stops early38 assert tr.protocol == "structured_debate" and tr.rounds == 239 assert len(tr.turns) == 12 * (1 + 2 * 2) # 6040 kinds = [(t.round, t.kind) for t in tr.turns]41 assert kinds.count((0, "estimate")) == 12 and kinds.count((1, "critique")) == 12 and kinds.count((2, "revision")) == 1242 assert all(t.critiques is not None for t in tr.turns if t.kind == "critique")43 assert len(tr.final_estimates) == 12 and set(RULES) <= set(tr.aggregated)44 # early stopping is recorded, not hidden45 tr2 = await structured_debate(packet, ROLES12, asg, pool, rounds=4, iqr_tol=10.0)46 assert tr2.rounds == 1 and any(n.startswith("stopped_early_at=1") for n in tr2.notes) and len(tr2.turns) == 36474849async def test_blinded_skeptic_and_verification(packet, pool):50 asg = assign(IDS, "all_openai")51 tr = await structured_debate(packet, ROLES12, asg, pool, rounds=1, blinded_skeptic=True, evidence_verification=True, arm="H_debate_skeptic", iqr_tol=0.0)52 sk = [t for t in tr.turns if t.role == "skeptic"]53 assert len(sk) == 1 and sk[0].round == 2 and sk[0].kind == "estimate" # one verdict after the rounds54 assert len(tr.turns) == 11 * 3 + 1 and len(tr.final_estimates) == 1255 assert any(n.startswith("blinded_skeptic_verdict") for n in tr.notes)56 assert any(n.startswith("verification_flags") for n in tr.notes) # the mock's building_condition cites M1 outside its view575859async def test_placebo_and_cost_matched(packet, pool):60 pr = placebo_roles(12)61 tr = await structured_debate(packet, pr, assign([r.id for r in pr], "all_anthropic"), pool, rounds=1, placebo=True, arm="I_placebo", iqr_tol=0.0)62 assert tr.protocol == "placebo_debate" and len(tr.turns) == 36 and set(tr.final_estimates) == {f"placebo_{k}" for k in range(1, 13)}63 tj = await cost_matched_single(packet, 12, {"generalist": "gpt-6.1-sol"}, pool, committee="all_openai")64 assert len(tj.turns) == 14 and set(tj.final_estimates) == {f"sample_{k:02d}" for k in range(1, 13)} | {"self_refine"}65 assert {"samples_mean", "samples_median", "self_refine"} <= set(tj.aggregated)66 assert tj.turns[12].kind == "critique" and tj.turns[13].kind == "revision"676869class _Failing(MockProvider):70 async def _raw_complete(self, **kw):71 raise RuntimeError("simulated outage")727374class _Garbage(MockProvider):75 async def _raw_complete(self, **kw):76 return '{"point_estimate": -1}', {"input_tokens": 10, "output_tokens": 3}777879async def test_failed_turns_are_not_fabricated(packet, pool):80 role = get_role("hedonic")81 for prov, expected in ((_Failing(ledger=pool.ledger), "RuntimeError"), (_Garbage(ledger=pool.ledger), "validation")):82 t = await run_turn(prov, role, "x <evidence_packet>{}</evidence_packet>", AgentEstimate, "claude-sonnet-5-5")83 assert t.estimate is None and t.error and expected in t.error84 pool._providers["claude-sonnet-5-5"] = _Failing(ledger=pool.ledger)85 tr = await independent(packet, ROLES12, assign(IDS, "all_anthropic"), pool)86 assert tr.final_estimates == {} and tr.aggregated == {} and all(t.error for t in tr.turns)878889async def test_spend_cap_propagates(packet, tmp_path):90 from agentic_valuation.agents.runner import ProviderPool91 from agentic_valuation.providers import CostLedger9293 p = ProviderPool(CostLedger(tmp_path / "l.jsonl", cap_usd=0.0001, run_id="cap"), force_mock=True)94 with pytest.raises(SpendCapExceeded):95 await independent(packet, ROLES12, assign(IDS, "all_anthropic"), p)969798def test_stopping_rule():99 assert iqr_log_points({"a": 100.0, "b": 110.0, "c": 120.0, "d": 130.0}) > 0100 assert not should_stop(None, 0.1) and should_stop(0.100, 0.1005) and not should_stop(0.100, 0.11) and should_stop(0.0, 0.0)101