# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca from __future__ import annotations import pytest from agentic_valuation.agents.roles import get_role, placebo_roles, roles_for from agentic_valuation.agents.runner import run_turn from agentic_valuation.debate.aggregation import RULES from agentic_valuation.debate.committees import assign from agentic_valuation.debate.protocols import cost_matched_single, independent, structured_debate from agentic_valuation.debate.stopping import iqr_log_points, should_stop from agentic_valuation.providers.ledger import SpendCapExceeded from agentic_valuation.providers.mock import MockProvider from agentic_valuation.schemas import AgentEstimate from tests.conftest_agents import packet, pool # noqa: F401 ROLES12 = roles_for("default12") IDS = [r.id for r in ROLES12] async def test_independent_12_estimates(packet, pool): asg = assign(IDS, "mixed_random", tx_id=packet.subject.tx_id) tr = await independent(packet, ROLES12, asg, pool, committee="mixed_random") assert len(tr.turns) == 12 and len(tr.final_estimates) == 12 and tr.rounds == 0 assert all(t.kind == "estimate" and t.round == 0 and t.error is None and t.estimate is not None for t in tr.turns) assert all(t.input_tokens > 0 and t.prompt_hash and t.model == asg[t.role] for t in tr.turns) assert set(RULES) <= set(tr.aggregated) and set(tr.aggregated_interval) == {"median_of_quantiles", "mixture"} assert tr.assignment == asg and tr.packet_hash == packet.content_hash() assert tr.total_tokens == sum(t.input_tokens + t.output_tokens for t in tr.turns) and tr.total_cost_usd > 0 # single generalist = arm E te = await independent(packet, [get_role("generalist")], {"generalist": "claude-sonnet-5-5"}, pool, arm="E_single_llm") assert list(te.final_estimates) == ["generalist"] and te.aggregated["median"] == pytest.approx(te.final_estimates["generalist"]) async def test_structured_debate_two_rounds(packet, pool): asg = assign(IDS, "all_anthropic") tr = await structured_debate(packet, ROLES12, asg, pool, rounds=2, iqr_tol=0.0) # tol 0 → never stops early assert tr.protocol == "structured_debate" and tr.rounds == 2 assert len(tr.turns) == 12 * (1 + 2 * 2) # 60 kinds = [(t.round, t.kind) for t in tr.turns] assert kinds.count((0, "estimate")) == 12 and kinds.count((1, "critique")) == 12 and kinds.count((2, "revision")) == 12 assert all(t.critiques is not None for t in tr.turns if t.kind == "critique") assert len(tr.final_estimates) == 12 and set(RULES) <= set(tr.aggregated) # early stopping is recorded, not hidden tr2 = await structured_debate(packet, ROLES12, asg, pool, rounds=4, iqr_tol=10.0) assert tr2.rounds == 1 and any(n.startswith("stopped_early_at=1") for n in tr2.notes) and len(tr2.turns) == 36 async def test_blinded_skeptic_and_verification(packet, pool): asg = assign(IDS, "all_openai") tr = await structured_debate(packet, ROLES12, asg, pool, rounds=1, blinded_skeptic=True, evidence_verification=True, arm="H_debate_skeptic", iqr_tol=0.0) sk = [t for t in tr.turns if t.role == "skeptic"] assert len(sk) == 1 and sk[0].round == 2 and sk[0].kind == "estimate" # one verdict after the rounds assert len(tr.turns) == 11 * 3 + 1 and len(tr.final_estimates) == 12 assert any(n.startswith("blinded_skeptic_verdict") for n in tr.notes) assert any(n.startswith("verification_flags") for n in tr.notes) # the mock's building_condition cites M1 outside its view async def test_placebo_and_cost_matched(packet, pool): pr = placebo_roles(12) tr = await structured_debate(packet, pr, assign([r.id for r in pr], "all_anthropic"), pool, rounds=1, placebo=True, arm="I_placebo", iqr_tol=0.0) assert tr.protocol == "placebo_debate" and len(tr.turns) == 36 and set(tr.final_estimates) == {f"placebo_{k}" for k in range(1, 13)} tj = await cost_matched_single(packet, 12, {"generalist": "gpt-6.1-sol"}, pool, committee="all_openai") assert len(tj.turns) == 14 and set(tj.final_estimates) == {f"sample_{k:02d}" for k in range(1, 13)} | {"self_refine"} assert {"samples_mean", "samples_median", "self_refine"} <= set(tj.aggregated) assert tj.turns[12].kind == "critique" and tj.turns[13].kind == "revision" class _Failing(MockProvider): async def _raw_complete(self, **kw): raise RuntimeError("simulated outage") class _Garbage(MockProvider): async def _raw_complete(self, **kw): return '{"point_estimate": -1}', {"input_tokens": 10, "output_tokens": 3} async def test_failed_turns_are_not_fabricated(packet, pool): role = get_role("hedonic") for prov, expected in ((_Failing(ledger=pool.ledger), "RuntimeError"), (_Garbage(ledger=pool.ledger), "validation")): t = await run_turn(prov, role, "x {}", AgentEstimate, "claude-sonnet-5-5") assert t.estimate is None and t.error and expected in t.error pool._providers["claude-sonnet-5-5"] = _Failing(ledger=pool.ledger) tr = await independent(packet, ROLES12, assign(IDS, "all_anthropic"), pool) assert tr.final_estimates == {} and tr.aggregated == {} and all(t.error for t in tr.turns) async def test_spend_cap_propagates(packet, tmp_path): from agentic_valuation.agents.runner import ProviderPool from agentic_valuation.providers import CostLedger p = ProviderPool(CostLedger(tmp_path / "l.jsonl", cap_usd=0.0001, run_id="cap"), force_mock=True) with pytest.raises(SpendCapExceeded): await independent(packet, ROLES12, assign(IDS, "all_anthropic"), p) def test_stopping_rule(): assert iqr_log_points({"a": 100.0, "b": 110.0, "c": 120.0, "d": 130.0}) > 0 assert not should_stop(None, 0.1) and should_stop(0.100, 0.1005) and not should_stop(0.100, 0.11) and should_stop(0.0, 0.0)