SPB Git

spb/wp26_uqo

Public
3.0 KB · 65 lines yaml
1# API-cost tiers. Costs are ESTIMATES from configs/models.yaml list prices and the token counts observed in the2# live sanity run of 2026-10-27 (single-agent turn at the mid tier: ~5.5k input + ~1.0k output tokens ≈ $0.02;3# F ≈ 12 turns ≈ $0.25; G2 ≈ 60 turns ≈ $1.2-1.4 per property). Actual spend is logged per call in results/ledger/.4# Self-imposed envelope for this version of the paper: ≈ 1,000 USD in total. Every run is resumable, so any tier can5# be extended later by raising `n` (the subsample order is fixed by the seed, so extensions are nested samples).6tier_default: mid7tiers:8  pilot:9    split: validation10    n: 1211    runs:12      - {arm: E_single_llm, committees: [all_anthropic, all_openai]}13      - {arm: F_independent, committees: [all_anthropic, all_openai]}14      - {arm: G_debate, rounds: 2, committees: [all_anthropic, all_openai]}15    cap_usd: 6016    purpose: "smoke on real validation data; prompt bugs; cost calibration"17  validation:18    split: validation19    n: 4020    runs:21      - {arm: E_single_llm, committees: [all_anthropic, all_openai, mixed_random]}22      - {arm: F_independent, committees: [all_anthropic, all_openai, mixed_random]}23      - {arm: G_debate, rounds: 2, committees: [all_anthropic, all_openai, mixed_random]}24      - {arm: G_debate, rounds: 1, committees: [mixed_random]}25      - {arm: G_debate, rounds: 4, committees: [mixed_random], n: 20}26      - {arm: H_debate_skeptic, rounds: 2, committees: [mixed_random]}27      - {arm: I_placebo, rounds: 2, committees: [mixed_random]}28      - {arm: J_cost_matched, committees: [mixed_random]}29      - {arm: G_fullview, rounds: 2, committees: [mixed_random]}30    cap_usd: 50031    purpose: "ablations, aggregation-weight estimation, protocol freeze"32  final:33    split: test34    n: 8035    runs:36      - {arm: E_single_llm, committees: [mixed_random]}37      - {arm: F_independent, committees: [mixed_random]}38      - {arm: G_debate, rounds: 2, committees: [mixed_random]}39      - {arm: H_debate_skeptic, rounds: 2, committees: [mixed_random]}40      - {arm: I_placebo, rounds: 2, committees: [mixed_random]}41      - {arm: J_cost_matched, committees: [mixed_random]}42    cap_usd: 50043    purpose: "test set, evaluated once after the freeze"44  final_small:45    split: test46    n: 30047    tier: small48    runs:49      - {arm: E_single_llm, committees: [mixed_random]}50      - {arm: F_independent, committees: [mixed_random]}51      - {arm: G_debate, rounds: 2, committees: [mixed_random]}52    cap_usd: 12053    purpose: "same protocol at the small cost-matched tier (claude-haiku-5-5 / gpt-6-luna): power + cost frontier"54  holdout:55    split: holdout56    n: 6057    runs:58      - {arm: E_single_llm, committees: [mixed_random]}59      - {arm: F_independent, committees: [mixed_random]}60      - {arm: G_debate, rounds: 2, committees: [mixed_random]}61    cap_usd: 12062    purpose: "untouched 2026 holdout, evaluated once"63scale_up_estimates_usd:64  note: "list-price estimates for extending the test tier (mid tier, mixed committee, six arms): n=200 ≈ 1,100; n=600 ≈ 3,300"65
Files · main
Menu

Theme