Repository Public
1/**2 * Opt-in live suite: `AOS_LIVE_TESTS=1 pnpm --filter @aos/providers test`.3 * Volume models only, tiny prompts, a few cents in total. Keys come from the environment or from4 * ~/.config/agent-os/dev.env (read here, never printed).5 */6import { existsSync, readFileSync, writeFileSync } from 'node:fs';7import { homedir, tmpdir } from 'node:os';8import { join } from 'node:path';9import Anthropic from '@anthropic-ai/sdk';10import type { Provider, ToolResult } from '@aos/shared';11import { afterAll, describe, expect, it } from 'vitest';12import { CATALOGUE, buildAnthropicRequest, createAnthropicAdapter, createOpenAIAdapter, promptTokens, type ModelDelta, type ProviderAdapter, type ToolDef } from './index.js';1314const LIVE = process.env.AOS_LIVE_TESTS === '1';1516if (LIVE) {17 const envFile = join(homedir(), '.config', 'agent-os', 'dev.env');18 if (existsSync(envFile)) {19 for (const line of readFileSync(envFile, 'utf8').split('\n')) {20 const m = /^\s*(?:export\s+)?([A-Z0-9_]+)=(.*)$/.exec(line);21 if (!m || process.env[m[1] as string]) continue;22 process.env[m[1] as string] = (m[2] as string).trim().replace(/^(['"])(.*)\1$/, '$2');23 }24 }25}2627const WEATHER: ToolDef = {28 name: 'get_weather',29 description: 'Get the current weather for a city. Call this whenever the user asks about the weather.',30 inputSchema: {31 type: 'object',32 properties: {33 city: { type: 'string', description: 'City name, e.g. Paris' },34 unit: { type: 'string', enum: ['celsius', 'fahrenheit'], description: 'Temperature unit (optional)' },35 },36 required: ['city'],37 },38};3940// Deterministic filler so the static prefix is long enough to be cached (≥ 512 tokens on Claude Haiku 5.5,41// ≥ 1 024 on GPT-6). No timestamps, no random values: the prefix must be byte-identical between calls.42const LONG_SYSTEM = [43 'You are a terse assistant used by an automated integration test. Follow the user instruction exactly.',44 ...Array.from({ length: 160 }, (_, i) => `Reference note ${i + 1}: the warehouse shelf ${i + 1} holds ${(i * 37) % 101} crates of item ${String.fromCharCode(65 + (i % 26))}${i}.`),45].join('\n');4647let spentUsd = 0;48const report: string[] = [];4950const cases: { provider: Provider; model: string; make: () => ProviderAdapter }[] = [51 // AOS_LIVE_ANTHROPIC_MODEL lets the owner point the suite at another model deliberately (default: the volume model).52 { provider: 'anthropic', model: process.env.AOS_LIVE_ANTHROPIC_MODEL ?? 'claude-haiku-5-5', make: () => createAnthropicAdapter() },53 { provider: 'openai', model: 'gpt-6-luna', make: () => createOpenAIAdapter() },54];5556describe.skipIf(!LIVE)('live providers (volume models)', () => {57 afterAll(() => {58 const text = `[live] ${report.join('\n[live] ')}\n[live] total estimated cost: USD ${spentUsd.toFixed(6)}\n`;59 console.log(`\n${text}`);60 // Reporters may hide console output of passing tests; keep a copy (usage numbers and model IDs only, no secrets).61 writeFileSync(join(tmpdir(), 'aos-providers-live-report.txt'), text);62 });6364 // Free (no generation): the token-counting endpoint applies the same request validation as /v1/messages, so65 // it checks the exact tools / system / thinking / effort / messages payload the adapter builds, including a66 // full tool round-trip and every operator-note placement.67 describe('anthropic request validation through count_tokens', () => {68 const adapter = createAnthropicAdapter();69 const model = 'claude-haiku-5-5';70 const count = async (transcript: unknown[]): Promise<number> => {71 const { params } = buildAnthropicRequest({ model, effort: 'low', system: LONG_SYSTEM, tools: [WEATHER], transcript });72 const { max_tokens: _maxTokens, ...countable } = params;73 const client = new Anthropic({ maxRetries: 2 });74 const res = await client.messages.countTokens(countable as unknown as Anthropic.Messages.MessageCountTokensParams);75 return res.input_tokens;76 };77 const toolTurn = (id: string) => [{78 role: 'assistant',79 content: [{ type: 'text', text: 'Checking the weather.' }, { type: 'tool_use', id, name: 'get_weather', input: { city: 'Paris', unit: null } }],80 }];81 const results = (id: string, ok: boolean): ToolResult[] => [{ callId: id, name: 'get_weather', ok, content: ok ? '{"temp_c":18}' : 'upstream timeout', artifact: null, trust: 'tool' }];8283 it('accepts the first-turn payload and a full tool round-trip with an error result', async () => {84 const t0 = adapter.initTranscript('What is the weather in Paris?');85 const n0 = await count(t0);86 const t1 = adapter.appendToolResults(adapter.appendAssistant(t0, toolTurn('toolu_01AosFixtureA')), results('toolu_01AosFixtureA', true));87 const t2 = adapter.appendToolResults(adapter.appendAssistant(t1, toolTurn('toolu_01AosFixtureB')), results('toolu_01AosFixtureB', false));88 const n2 = await count(t2);89 expect(n0).toBeGreaterThan(512); // long enough for the cache minimum90 expect(n2).toBeGreaterThan(n0);91 report.push(`anthropic count_tokens: first turn=${n0} tokens, after two tool round-trips=${n2}`);92 }, 60_000);9394 it('accepts every operator-note placement the adapter can render', async () => {95 const t0 = adapter.initTranscript('What is the weather in Paris?');96 const afterTools = adapter.appendToolResults(adapter.appendAssistant(t0, toolTurn('toolu_01AosFixtureC')), results('toolu_01AosFixtureC', true));97 const placements: Record<string, unknown[]> = {98 'system after tool results (last)': adapter.appendOperatorNote(afterTools, 'Wrap up now.'),99 'two notes merged': adapter.appendOperatorNote(adapter.appendOperatorNote(afterTools, 'Wrap up now.'), 'Be brief.'),100 'system followed by assistant': adapter.appendUser(adapter.appendAssistant(adapter.appendOperatorNote(afterTools, 'Wrap up now.'), [{ role: 'assistant', content: [{ type: 'text', text: 'It is 18°C.' }] }]), 'Thanks. And tomorrow?'),101 'note after an assistant turn (user-turn reminder)': adapter.appendOperatorNote(adapter.appendAssistant(t0, [{ role: 'assistant', content: [{ type: 'text', text: 'I am not sure.' }] }]), 'Continue or call finish.'),102 'note then steering user message (user-turn reminder)': adapter.appendUser(adapter.appendOperatorNote(afterTools, 'Wrap up now.'), 'Also give Fahrenheit.'),103 };104 for (const [name, transcript] of Object.entries(placements)) {105 expect(await count(transcript), name).toBeGreaterThan(0);106 }107 report.push(`anthropic count_tokens: ${Object.keys(placements).length} operator-note placements accepted`);108 }, 120_000);109110 it('is a real validator: a system message in first position is rejected', async () => {111 const err = await count([{ role: 'system', content: 'x' }, { role: 'user', content: [{ type: 'text', text: 'hi' }] }]).catch((e: unknown) => e);112 // The adapter itself never renders this (a leading note becomes a user-turn reminder), so bypass it:113 expect(err).not.toBeInstanceOf(Error); // rendered as a reminder ⇒ accepted114 const client = new Anthropic({ maxRetries: 2 });115 const raw = await client.messages.countTokens({116 model, messages: [{ role: 'system', content: 'x' }, { role: 'user', content: 'hi' }],117 } as unknown as Anthropic.Messages.MessageCountTokensParams).catch((e: unknown) => e);118 expect(raw).toBeInstanceOf(Anthropic.BadRequestError);119 }, 60_000);120 });121122 for (const { provider, model, make } of cases) {123 describe(provider, () => {124 const adapter = make();125126 it('answers a plain text turn with streaming deltas', async () => {127 const deltas: ModelDelta[] = [];128 const res = await adapter.call({129 model, effort: 'low', system: 'You are a terse assistant.', tools: [],130 transcript: adapter.initTranscript('Reply with exactly the word: pong'),131 maxOutputTokens: 2000, onDelta: (d) => deltas.push(d),132 });133 spentUsd += res.usage.usd;134 const text = res.blocks.filter((b) => b.type === 'text').map((b) => b.text).join('');135 expect(res.stop).toBe('end_turn');136 expect(text.toLowerCase()).toContain('pong');137 expect(deltas.filter((d) => d.type === 'text').map((d) => d.text).join('')).toBe(text);138 expect(res.usage.inputTokens + res.usage.cacheReadTokens + res.usage.cacheWriteTokens).toBeGreaterThan(0);139 expect(res.usage.outputTokens).toBeGreaterThan(0);140 expect(res.usage.usd).toBeGreaterThan(0);141 expect(res.usage.usd).toBeCloseTo(adapter.price(model, res.usage), 9);142 expect(res.native.length).toBeGreaterThan(0);143 report.push(`${provider} text: model=${res.model} stop=${res.nativeStop} usage=${JSON.stringify(res.usage)} reasoningDeltas=${deltas.filter((d) => d.type === 'reasoning').length}`);144 }, 120_000);145146 it('completes a tool round-trip with transcript continuity, an operator note and a cache hit', async () => {147 const base = { model, effort: 'low' as const, system: LONG_SYSTEM, tools: [WEATHER], maxOutputTokens: 4000 };148 let transcript = adapter.initTranscript('What is the weather in Paris right now? Use the get_weather tool, then answer.');149 const first = await adapter.call({ ...base, transcript });150 spentUsd += first.usage.usd;151 expect(first.stop).toBe('tool_use');152 const calls = first.blocks.filter((b) => b.type === 'tool_call');153 expect(calls.length).toBeGreaterThanOrEqual(1);154 expect(calls[0]?.name).toBe('get_weather');155 const input = calls[0]?.input as { city?: string; unit?: unknown };156 expect(String(input.city).toLowerCase()).toContain('paris');157 // An optional property the model left out arrives as absent, never as null.158 expect(input.unit === undefined || typeof input.unit === 'string').toBe(true);159160 const before = transcript.length;161 transcript = adapter.appendAssistant(transcript, first.native);162 const results: ToolResult[] = calls.map((c) => ({163 callId: c.id, name: c.name, ok: true, content: '{"city":"Paris","temp_c":18,"sky":"overcast"}', artifact: null, trust: 'tool',164 }));165 transcript = adapter.appendToolResults(transcript, results);166 transcript = adapter.appendOperatorNote(transcript, 'Operator instruction: answer in one short sentence and end your answer with the word DONE.');167 expect(transcript.length).toBeGreaterThan(before + 2);168169 const deltas: ModelDelta[] = [];170 const second = await adapter.call({ ...base, transcript, onDelta: (d) => deltas.push(d) });171 spentUsd += second.usage.usd;172 const text = second.blocks.filter((b) => b.type === 'text').map((b) => b.text).join('');173 expect(second.stop).toBe('end_turn');174 expect(text).toContain('18');175 expect(text).toContain('DONE');176 expect(deltas.some((d) => d.type === 'text')).toBe(true);177 for (const r of [first, second]) {178 expect(promptTokens(r.usage)).toBeGreaterThan(0);179 expect(r.usage.outputTokens).toBeGreaterThan(0);180 expect(r.usage.usd).toBeGreaterThan(0);181 }182 report.push(`${provider} tool#1: stop=${first.nativeStop} usage=${JSON.stringify(first.usage)}`);183 report.push(`${provider} tool#2: stop=${second.nativeStop} usage=${JSON.stringify(second.usage)} text=${JSON.stringify(text.slice(0, 120))}`);184 if (provider === 'anthropic') {185 // Identical tools + system prefix: the second request must read the prefix written by the first.186 expect(first.usage.cacheReadTokens + first.usage.cacheWriteTokens).toBeGreaterThan(0);187 expect(second.usage.cacheReadTokens).toBeGreaterThan(0);188 }189 }, 180_000);190191 it('returns reasoning summaries at a higher effort (reported, not required)', async () => {192 const deltas: ModelDelta[] = [];193 const res = await adapter.call({194 model, effort: 'high', system: 'You are a careful assistant.', tools: [],195 transcript: adapter.initTranscript('How many prime numbers are there between 100 and 150? Reply with the number only.'),196 maxOutputTokens: 6000, onDelta: (d) => deltas.push(d),197 });198 spentUsd += res.usage.usd;199 expect(res.stop).toBe('end_turn');200 expect(res.blocks.filter((b) => b.type === 'text').map((b) => b.text).join('')).toContain('10');201 const summaries = res.blocks.filter((b) => b.type === 'reasoning').map((b) => b.summary);202 const streamed = deltas.filter((d) => d.type === 'reasoning').map((d) => d.text).join('');203 // Whatever was streamed as reasoning is exactly what the reasoning blocks contain.204 expect(streamed.replace(/\s+/g, '')).toBe(summaries.join('').replace(/\s+/g, ''));205 report.push(`${provider} reasoning: blocks=${summaries.length} summaryChars=${summaries.join('').length} streamedChars=${streamed.length} usage=${JSON.stringify(res.usage)}`);206 }, 180_000);207208 it('lists catalogue availability from the live Models API', async () => {209 const [models, liveIds] = await Promise.all([adapter.listModels(), adapter.listLiveModelIds()]);210 const mine = CATALOGUE.filter((m) => m.provider === provider).map((m) => m.id);211 expect(models.map((m) => m.id)).toEqual(mine);212 for (const m of models) expect(m.available).toBe(liveIds.includes(m.id));213 expect(models.find((m) => m.id === model)?.available).toBe(true);214 report.push(`${provider} availability: ${models.map((m) => `${m.id}=${m.available}(ctx ${m.contextTokens ?? '—'}, out ${m.maxOutputTokens ?? '—'})`).join(', ')}`);215 const family = liveIds.filter((id) => (provider === 'openai' ? /^gpt-6/.test(id) : /^claude-/.test(id))).sort();216 report.push(`${provider} live IDs (${liveIds.length} total): ${family.join(', ')}`);217 }, 60_000);218 });219 }220});221