1"""LLM agent laboratory (Model C with live local models).23Cells (pre-registered): policy ∈ {bayesian control, surrogate (rule-based mock with the LLM interface), single shared4LLM (qwen7b), diverse LLMs (qwen7b / mistral7b / llama3b assigned round-robin), shared LLM with varied prompts},5signal structure ∈ {private (D=0), shared model (D=1, M=1), diverse models (D=1, M=3)}, temperature ∈ {0, 0.7},6seeds 1..S. Design: CDA. All calls cached and audited (results/llm_audit/*.jsonl); invalid actions counted.7Run: python3 scripts/35_llm_lab.py [--quick] [--providers qwen7b,mistral7b,llama3b]8Outputs: results/llm_lab/cells.csv, results/llm_lab/summary.csv, results/llm_lab/audit_stats.json9"""10from __future__ import annotations1112import json13import sys14import time15from pathlib import Path1617import numpy as np18import pandas as pd1920sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))21from age.llm.client import LLMClient, LOCAL_PROVIDERS, MockLLM # noqa: E40222from age.markets.simulation import MarketConfig, run_market # noqa: E4022324ROOT = Path(__file__).resolve().parents[1]25OUT = ROOT / 'results' / 'llm_lab'26OUT.mkdir(parents=True, exist_ok=True)27QUICK = '--quick' in sys.argv28N, T, BURN = (8, 24, 4) if QUICK else (10, 40, 8)29SEEDS = range(1, 4) if QUICK else range(1, 7)30provs = ['qwen7b', 'mistral7b', 'llama3b']31for a in sys.argv:32 if a.startswith('--providers'):33 provs = a.split('=')[1].split(',')3435clients = {k: LLMClient(LOCAL_PROVIDERS[k], run_id=f'lab_{k}') for k in provs}36clients['mock'] = MockLLM(run_id='lab_mock')37KEYS = ['vol', 'of_corr', 'of_corr_deleg', 'pressure', 'r2_common', 'n_trades', 'invalid', 'clipped', 'defaults', 'leverage']383940def run_cell(policy, D, M, temp, seed, memory=0):41 kw = dict(n_agents=N, T=T, burn_in=BURN, design='cda', delegation_share=D, M=M, seed=seed, llm_temperature=temp,42 llm_memory=memory)43 if policy == 'bayesian':44 cfg = MarketConfig(policy='bayesian', **kw)45 elif policy == 'surrogate':46 cfg = MarketConfig(policy='surrogate', llm_clients={'mock': clients['mock']}, llm_assignment=['mock'] * N, **kw)47 elif policy == 'llm_shared':48 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N, **kw)49 elif policy == 'llm_diverse':50 assign = [provs[i % len(provs)] for i in range(N)]51 cfg = MarketConfig(policy='llm_mix', llm_clients={k: clients[k] for k in provs}, llm_assignment=assign, **kw)52 elif policy == 'llm_shared_varied_prompts':53 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N,54 llm_variants=[i % 5 for i in range(N)], **kw)55 elif policy == 'llm_shared_memory':56 cfg = MarketConfig(policy='llm', llm_clients={provs[0]: clients[provs[0]]}, llm_assignment=[provs[0]] * N, **kw)57 cfg.llm_memory = 358 else:59 raise ValueError(policy)60 t0 = time.time()61 out = run_market(cfg)62 row = dict(policy=policy, D=D, M=M, temperature=temp, seed=seed, runtime_s=time.time() - t0)63 row.update({k: out.get(k, np.nan) for k in KEYS})64 # action homogeneity: average pairwise correlation of *raw* order flow among agents (not residualised)65 of = out['order_flow'][BURN:]66 with np.errstate(invalid='ignore', divide='ignore'):67 C = np.nan_to_num(np.corrcoef(of.T))68 row['raw_of_corr'] = float(C[np.triu_indices(N, 1)].mean())69 row['price_end'] = float(out['prices'][-1])70 return row717273rows = []74cells = []75for policy in ['bayesian', 'surrogate', 'llm_shared', 'llm_diverse', 'llm_shared_varied_prompts']:76 for (D, M) in [(0.0, 1), (1.0, 1), (1.0, 3)]:77 temps = [0.0] if policy in ('bayesian', 'surrogate') else [0.0, 0.7]78 for temp in temps:79 for s in SEEDS:80 cells.append((policy, D, M, temp, s))81if not QUICK:82 for s in SEEDS:83 cells.append(('llm_shared_memory', 1.0, 1, 0.0, s))84print(f'{len(cells)} cells, N={N}, T={T}')85for i, c in enumerate(cells):86 r = run_cell(*c)87 rows.append(r)88 print(f"[{i+1}/{len(cells)}] {c} vol={r['vol']:.4f} ofc={r['of_corr']:.2f} raw={r['raw_of_corr']:.2f} trades={r['n_trades']:.1f} invalid={r['invalid']} ({r['runtime_s']:.0f}s)", flush=True)89 pd.DataFrame(rows).to_csv(OUT / 'cells.csv', index=False)90df = pd.DataFrame(rows)91summ = df.groupby(['policy', 'D', 'M', 'temperature'])[KEYS + ['raw_of_corr', 'runtime_s']].agg(['mean', 'std']).reset_index()92summ.columns = ['_'.join(c).strip('_') for c in summ.columns]93summ.to_csv(OUT / 'summary.csv', index=False)94stats = {k: v.stats() for k, v in clients.items()}95json.dump(stats, open(OUT / 'audit_stats.json', 'w'), indent=1)96print(summ.to_string())97print(stats)98