1"""LLM agent laboratory (Model C with live local models).23Cells (pre-registered): policy ∈ {bayesian control, surrogate (rule-based mock with the LLM interface), single shared4LLM (qwen7b), diverse LLMs (qwen7b / mistral7b / llama3b assigned round-robin), shared LLM with varied prompts},5signal structure ∈ {private (D=0), shared model (D=1, M=1), diverse models (D=1, M=3)}, temperature ∈ {0, 0.7},6seeds 1..S. Design: CDA. All calls cached and audited (results/llm_audit/*.jsonl); invalid actions counted.7Run: python3 scripts/35_llm_lab.py [--quick] [--providers qwen7b,mistral7b,llama3b]8Outputs: results/llm_lab/cells.csv, results/llm_lab/summary.csv, results/llm_lab/audit_stats.json9"""10from __future__ import annotations1112import json13import sys14import time15from pathlib import Path1617import numpy as np18import pandas as pd1920sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))21from age.llm.client import LLMClient, LOCAL_PROVIDERS, MockLLM # noqa: E40222from age.markets.simulation import MarketConfig, run_market # noqa: E4022324ROOT = Path(__file__).resolve().parents[1]25OUT = ROOT / 'results' / 'llm_lab'26OUT.mkdir(parents=True, exist_ok=True)27QUICK = '--quick' in sys.argv28N, T, BURN = (8, 24, 4) if QUICK else (10, 40, 8)29SEEDS = range(1, 4) if QUICK else range(1, 7)30provs = ['qwen7b', 'mistral7b', 'llama3b']31POLICIES = ['bayesian', 'surrogate', 'llm_shared', 'llm_diverse', 'llm_shared_varied_prompts']32TAG = ''33for a in sys.argv:34 if a.startswith('--providers'):35 provs = a.split('=')[1].split(',')36 if a.startswith('--policies='):37 POLICIES = a.split('=')[1].split(',')38 if a.startswith('--tag='):39 TAG = '_' + a.split('=')[1]4041clients = {k: LLMClient(LOCAL_PROVIDERS[k], run_id=f'lab_{k}') for k in provs}42clients['mock'] = MockLLM(run_id='lab_mock')43KEYS = ['vol', 'of_corr', 'of_corr_deleg', 'pressure', 'r2_common', 'n_trades', 'invalid', 'clipped', 'defaults', 'leverage']444546def run_cell(policy, D, M, temp, seed, memory=0):47 kw = dict(n_agents=N, T=T, burn_in=BURN, design='cda', delegation_share=D, M=M, seed=seed, llm_temperature=temp,48 llm_memory=memory)49 if policy == 'bayesian':50 cfg = MarketConfig(policy='bayesian', **kw)51 elif policy == 'surrogate':52 cfg = MarketConfig(policy='surrogate', llm_clients={'mock': clients['mock']}, llm_assignment=['mock'] * N, **kw)53 elif policy == 'llm_shared':54 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N, **kw)55 elif policy == 'llm_diverse':56 assign = [provs[i % len(provs)] for i in range(N)]57 cfg = MarketConfig(policy='llm_mix', llm_clients={k: clients[k] for k in provs}, llm_assignment=assign, **kw)58 elif policy == 'llm_shared_varied_prompts':59 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N,60 llm_variants=[i % 5 for i in range(N)], **kw)61 elif policy == 'llm_shared_memory':62 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N, **kw)63 cfg.llm_memory = 364 else:65 raise ValueError(policy)66 t0 = time.time()67 out = run_market(cfg)68 row = dict(policy=policy, D=D, M=M, temperature=temp, seed=seed, runtime_s=time.time() - t0)69 row.update({k: out.get(k, np.nan) for k in KEYS})70 # action homogeneity: average pairwise correlation of *raw* order flow among agents (not residualised)71 of = out['order_flow'][BURN:]72 with np.errstate(invalid='ignore', divide='ignore'):73 C = np.nan_to_num(np.corrcoef(of.T))74 row['raw_of_corr'] = float(C[np.triu_indices(N, 1)].mean())75 row['price_end'] = float(out['prices'][-1])76 return row777879rows = []80cells = []81for policy in POLICIES:82 for (D, M) in [(0.0, 1), (1.0, 1), (1.0, 3)]:83 temps = [0.0] if policy in ('bayesian', 'surrogate') else [0.0, 0.7]84 for temp in temps:85 for s in SEEDS:86 cells.append((policy, D, M, temp, s))87if not QUICK:88 for s in SEEDS:89 cells.append(('llm_shared_memory', 1.0, 1, 0.0, s))90print(f'{len(cells)} cells, N={N}, T={T}')91for i, c in enumerate(cells):92 r = run_cell(*c)93 rows.append(r)94 print(f"[{i+1}/{len(cells)}] {c} vol={r['vol']:.4f} ofc={r['of_corr']:.2f} raw={r['raw_of_corr']:.2f} trades={r['n_trades']:.1f} invalid={r['invalid']} ({r['runtime_s']:.0f}s)", flush=True)95 pd.DataFrame(rows).to_csv(OUT / f'cells{TAG}.csv', index=False)96df = pd.DataFrame(rows)97summ = df.groupby(['policy', 'D', 'M', 'temperature'])[KEYS + ['raw_of_corr', 'runtime_s']].agg(['mean', 'std']).reset_index()98summ.columns = ['_'.join(c).strip('_') for c in summ.columns]99summ.to_csv(OUT / f'summary{TAG}.csv', index=False)100stats = {k: v.stats() for k, v in clients.items()}101json.dump(stats, open(OUT / f'audit_stats{TAG}.json', 'w'), indent=1)102print(summ.to_string())103print(stats)104