1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Deliberation protocols. Each coroutine returns a DebateTranscript for ONE property.34F independent(...) round 0 only, no communication (E = same with roles=[generalist]).5G structured_debate(...) round 0 independent → for each round r: critiques (CritiqueSet) then revised6 estimates (AgentEstimate) with the anonymised committee state; pre-specified7 stopping when the IQR of log points moves < 1 % (recorded, rounds actually run8 reported in ``transcript.rounds``).9H structured_debate(..., blinded_skeptic=True, evidence_verification=True)10 the skeptic never joins the rounds; it sees the packet + final estimates only11 and its verdict is appended; deterministic verification flags are injected12 into the next prompt of each flagged agent.13I structured_debate(..., placebo=True) with placebo roles (same mechanics, generic / permuted prompts).14J cost_matched_single(...) generalist sampled k times independently + a 2-step self-refine chain.1516Agents of the same round run concurrently (asyncio.gather); the provider semaphore bounds the calls.17Failed turns (error set, estimate None) are excluded from the committee state and from aggregation.18"""19from __future__ import annotations2021import asyncio22import time23from typing import Any2425from ..agents.packets import committee_state, render_user_prompt, role_view, verify_evidence, visible_ids26from ..agents.roles import Role, get_role27from ..agents.runner import ProviderPool, run_turn28from ..schemas import AgentEstimate, AgentTurn, CritiqueSet, DebateTranscript, EvidencePacket29from .aggregation import aggregate_intervals, aggregate_points, interval_9030from .stopping import IQR_TOL, iqr_log_points, should_stop313233def _latest_estimates(turns: list[AgentTurn]) -> dict[str, AgentEstimate]:34 """role -> last successful estimate (estimate or revision kinds), in role order of first appearance."""35 out: dict[str, AgentEstimate] = {}36 for t in turns:37 if t.kind in ("estimate", "revision") and t.estimate is not None:38 out[t.role] = t.estimate39 return out404142def _finalize(tr: DebateTranscript, turns: list[AgentTurn], t0: float, weights: dict[str, Any] | None, final: dict[str, AgentEstimate] | None = None) -> DebateTranscript:43 tr.turns = turns44 est = final if final is not None else _latest_estimates(turns)45 tr.final_estimates = {r: e.point_estimate for r, e in est.items()}46 tr.final_quantiles = {r: [e.q05, e.q25, e.q50, e.q75, e.q95] for r, e in est.items()}47 tr.aggregated = aggregate_points(tr.final_estimates, weights)48 tr.aggregated_interval = interval_90(aggregate_intervals(tr.final_quantiles))49 tr.total_cost_usd = round(sum(t.cost_usd for t in turns), 6)50 tr.total_tokens = sum(t.input_tokens + t.output_tokens for t in turns)51 tr.total_latency_s = round(time.perf_counter() - t0, 3)52 return tr535455async def _gather_turns(coros: list[Any]) -> list[AgentTurn]:56 return list(await asyncio.gather(*coros))575859def _model_for(assignment: dict[str, str] | str, role_id: str) -> str:60 if isinstance(assignment, str):61 return assignment62 try:63 return assignment[role_id]64 except KeyError as e:65 raise KeyError(f"no model assigned to role {role_id!r}") from e666768# ---------------------------------------------------------------------------------------------------69# F / E70# ---------------------------------------------------------------------------------------------------71async def independent(72 packet: EvidencePacket,73 roles: list[Role],74 committee_assignment: dict[str, str],75 pool: ProviderPool,76 *,77 arm: str = "F_independent",78 committee: str = "all_anthropic",79 effort: str = "medium",80 max_tokens: int = 4096,81 weights: dict[str, Any] | None = None,82) -> DebateTranscript:83 t0 = time.perf_counter()84 tr = DebateTranscript(tx_id=packet.subject.tx_id, arm=arm, committee=committee, protocol="independent", rounds=0, packet_hash=packet.content_hash(), turns=[], assignment=dict(committee_assignment))85 turns = await _gather_turns(86 [87 run_turn(pool.get(_model_for(committee_assignment, r.id)), r, render_user_prompt(role_view(packet, r), None, kind="estimate"), AgentEstimate, _model_for(committee_assignment, r.id), effort, 0, "estimate", max_tokens=max_tokens)88 for r in roles89 ]90 )91 return _finalize(tr, turns, t0, weights)929394# ---------------------------------------------------------------------------------------------------95# G / H / I96# ---------------------------------------------------------------------------------------------------97async def structured_debate(98 packet: EvidencePacket,99 roles: list[Role],100 assignment: dict[str, str],101 pool: ProviderPool,102 rounds: int = 2,103 *,104 blinded_skeptic: bool = False,105 evidence_verification: bool = False,106 placebo: bool = False,107 arm: str = "G_debate",108 committee: str = "all_anthropic",109 effort: str = "medium",110 max_tokens: int = 4096,111 weights: dict[str, Any] | None = None,112 iqr_tol: float = IQR_TOL,113 skeptic_role_id: str = "skeptic",114) -> DebateTranscript:115 t0 = time.perf_counter()116 protocol = "placebo_debate" if placebo else ("debate_skeptic_verification" if (blinded_skeptic or evidence_verification) else "structured_debate")117 tr = DebateTranscript(tx_id=packet.subject.tx_id, arm=arm, committee=committee, protocol=protocol, rounds=rounds, packet_hash=packet.content_hash(), turns=[], assignment=dict(assignment))118 skeptic: Role | None = None119 debaters = list(roles)120 if blinded_skeptic:121 skeptic = next((r for r in roles if r.id == skeptic_role_id), None)122 if skeptic is None:123 skeptic = get_role(skeptic_role_id)124 tr.notes.append(f"blinded_skeptic: role {skeptic_role_id} added (not in role list)")125 debaters = [r for r in roles if r.id != skeptic.id]126 if skeptic.id not in assignment:127 raise KeyError(f"no model assigned to the blinded skeptic {skeptic.id!r}")128 views = {r.id: role_view(packet, r) for r in debaters}129 vis = {r.id: visible_ids(packet, r) for r in debaters}130 turns: list[AgentTurn] = []131 flags: dict[str, list[str]] = {}132133 def _verify(est: dict[str, AgentEstimate], round_no: int) -> None:134 if not evidence_verification:135 return136 flags.clear()137 for rid, e in est.items():138 f = verify_evidence(e, packet, allowed_ids=vis[rid])139 if f:140 flags[rid] = f141 tr.notes.append(f"verification_flags round={round_no} role={rid}: {';'.join(f)}")142143 # round 0 — independent144 turns += await _gather_turns(145 [run_turn(pool.get(_model_for(assignment, r.id)), r, render_user_prompt(views[r.id], None, kind="estimate"), AgentEstimate, _model_for(assignment, r.id), effort, 0, "estimate", max_tokens=max_tokens) for r in debaters]146 )147 current = _latest_estimates(turns)148 _verify(current, 0)149 prev_iqr = iqr_log_points({r: e.point_estimate for r, e in current.items()}) if current else None150 rounds_run = 0151 for r_no in range(1, rounds + 1):152 if len(current) < 2:153 tr.notes.append(f"stopped_early_at={r_no - 1}: fewer than two valid estimates")154 break155 # critiques: every debater sees the anonymised table156 crit_turns = await _gather_turns(157 [158 run_turn(pool.get(_model_for(assignment, r.id)), r, render_user_prompt(views[r.id], committee_state(r_no, current, for_role=r.id, visible=vis[r.id]), kind="critique", verification_flags=flags.get(r.id)), CritiqueSet, _model_for(assignment, r.id), effort, r_no, "critique", max_tokens=max_tokens)159 for r in debaters160 ]161 )162 turns += crit_turns163 addressed: dict[str, list[dict[str, Any]]] = {r.id: [] for r in debaters}164 for ct in crit_turns:165 if ct.critiques is None:166 continue167 for c in ct.critiques.critiques:168 if c.target_role in addressed and c.target_role != ct.role:169 addressed[c.target_role].append({"from": "committee_member", "agree": c.agree, "points": c.points[:6], "evidence_ids_cited": c.evidence_ids_cited, "suggested_value": c.suggested_value})170 # revisions171 rev_turns = await _gather_turns(172 [173 run_turn(pool.get(_model_for(assignment, r.id)), r, render_user_prompt(views[r.id], committee_state(r_no, current, for_role=r.id, critiques=addressed[r.id], visible=vis[r.id]), kind="revision", verification_flags=flags.get(r.id)), AgentEstimate, _model_for(assignment, r.id), effort, r_no, "revision", max_tokens=max_tokens)174 for r in debaters175 ]176 )177 turns += rev_turns178 rounds_run = r_no179 current = _latest_estimates(turns)180 _verify(current, r_no)181 cur_iqr = iqr_log_points({r: e.point_estimate for r, e in current.items()})182 if should_stop(prev_iqr, cur_iqr, iqr_tol) and r_no < rounds:183 tr.notes.append(f"stopped_early_at={r_no}: IQR(log points) {prev_iqr:.5f} -> {cur_iqr:.5f} (< {iqr_tol:.0%} change); pre-specified rounds={rounds}")184 prev_iqr = cur_iqr185 break186 prev_iqr = cur_iqr187 tr.rounds = rounds_run188 if flags:189 tr.notes.append("verification_flags_unresolved_after_final_round: " + ",".join(sorted(flags)))190 # blinded skeptic verdict (H): packet + final estimates only, never the critiques191 if skeptic is not None:192 sk_view = role_view(packet, skeptic)193 state = committee_state(rounds_run + 1, current)194 state.pop("critiques_addressed_to_you", None)195 sk_turn = await run_turn(pool.get(_model_for(assignment, skeptic.id)), skeptic, render_user_prompt(sk_view, state, kind="skeptic_verdict"), AgentEstimate, _model_for(assignment, skeptic.id), effort, rounds_run + 1, "estimate", max_tokens=max_tokens)196 turns.append(sk_turn)197 tr.notes.append("blinded_skeptic_verdict: appended as final estimate of role 'skeptic' (round %d), discussion not shown" % (rounds_run + 1))198 return _finalize(tr, turns, t0, weights)199200201# ---------------------------------------------------------------------------------------------------202# J203# ---------------------------------------------------------------------------------------------------204async def cost_matched_single(205 packet: EvidencePacket,206 k: int,207 assignment: dict[str, str] | str,208 pool: ProviderPool,209 *,210 arm: str = "J_cost_matched",211 committee: str = "all_anthropic",212 effort: str = "medium",213 max_tokens: int = 4096,214 weights: dict[str, Any] | None = None,215 role_id: str = "generalist",216) -> DebateTranscript:217 """k independent generalist samples (same prompt; vendor-side sampling provides the variation — the218 mock is deterministic and returns identical samples) + a 2-step self-refine chain (self-critique then219 refined estimate) seeded by the first sample. Final estimates: sample_01..sample_k and self_refine."""220 t0 = time.perf_counter()221 role = get_role(role_id)222 model = _model_for(assignment, role.id)223 tr = DebateTranscript(tx_id=packet.subject.tx_id, arm=arm, committee=committee, protocol="cost_matched_single", rounds=0, packet_hash=packet.content_hash(), turns=[], assignment={role.id: model})224 view = role_view(packet, role)225 prompt = render_user_prompt(view, None, kind="estimate")226 samples = await _gather_turns([run_turn(pool.get(model), role, prompt, AgentEstimate, model, effort, 0, "estimate", max_tokens=max_tokens, tag=f"{role.id}:sample{i + 1}") for i in range(k)])227 turns: list[AgentTurn] = []228 final: dict[str, AgentEstimate] = {}229 for i, t in enumerate(samples):230 t.role = f"sample_{i + 1:02d}"231 turns.append(t)232 if t.estimate is not None:233 final[t.role] = t.estimate234 seed_est = next((t.estimate for t in samples if t.estimate is not None), None)235 if seed_est is not None:236 state = committee_state(1, {role.id: seed_est}, for_role=role.id)237 crit = await run_turn(pool.get(model), role, render_user_prompt(view, state, kind="self_critique"), CritiqueSet, model, effort, 1, "critique", max_tokens=max_tokens, tag=f"{role.id}:self_critique")238 crit.role = "self_refine"239 turns.append(crit)240 own = []241 if crit.critiques is not None:242 own = [{"from": "self", "agree": c.agree, "points": c.points[:6], "evidence_ids_cited": c.evidence_ids_cited, "suggested_value": c.suggested_value} for c in crit.critiques.critiques]243 state2 = committee_state(2, {role.id: seed_est}, for_role=role.id, critiques=own)244 ref = await run_turn(pool.get(model), role, render_user_prompt(view, state2, kind="self_refine"), AgentEstimate, model, effort, 2, "revision", max_tokens=max_tokens, tag=f"{role.id}:self_refine")245 ref.role = "self_refine"246 turns.append(ref)247 if ref.estimate is not None:248 final["self_refine"] = ref.estimate249 tr.rounds = 2250 else:251 tr.notes.append("self_refine skipped: no valid sample to seed the chain")252 tr.notes.append(f"k={k} independent samples of role {role.id}; self_refine = 2-step chain")253 out = _finalize(tr, turns, t0, weights, final=final)254 # convenience: the two cost-matched comparators the paper reports255 if final:256 samp = {r: e.point_estimate for r, e in final.items() if r.startswith("sample_")}257 if samp:258 out.aggregated["samples_mean"] = aggregate_points(samp)["mean"]259 out.aggregated["samples_median"] = aggregate_points(samp)["median"]260 if "self_refine" in final:261 out.aggregated["self_refine"] = final["self_refine"].point_estimate262 return out263264265PROTOCOLS = {"independent": independent, "structured_debate": structured_debate, "cost_matched_single": cost_matched_single}266