Repository Public
1import type { ToolResult } from '@aos/shared';2import { describe, expect, it } from 'vitest';3import { ProviderError } from './errors.js';4import { createOpenAIAdapter, openaiItems, openaiUsage } from './openai.js';5import { invalidToolInputMessage, parseInvalidToolInput } from './schema.js';6import {7 fakeFetch, hangUntilAborted, jsonResponse, noSleep, openaiError, openaiEvents, openaiStream, sseResponse,8 type Responder,9} from './test/fixtures.js';10import type { ModelDelta, ModelRequest, ToolDef } from './types.js';1112const READ: ToolDef = {13 name: 'read_file', description: 'Read a file',14 inputSchema: { type: 'object', properties: { path: { type: 'string' }, limit: { type: 'integer', minimum: 1 } }, required: ['path'] },15};16const LIST: ToolDef = {17 name: 'list_dir', description: 'List a directory',18 inputSchema: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },19};2021const reasoning = (id: string, summary: string[]) => ({22 id, type: 'reasoning', summary: summary.map((text) => ({ type: 'summary_text', text })), encrypted_content: `enc-fixture-${id}`, status: 'completed',23});24const message = (id: string, text: string) => ({25 id, type: 'message', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text, annotations: [] }],26});27const call = (id: string, callId: string, name: string, args: string, status = 'completed') => ({28 id, type: 'function_call', call_id: callId, name, arguments: args, status,29});3031function setup(responders: Responder[], opts: { maxAttempts?: number } = {}) {32 const { fetch, requests } = fakeFetch(responders);33 const { sleep, waits } = noSleep();34 const adapter = createOpenAIAdapter({ apiKey: 'test-key-not-real', fetch, sleep, random: () => 0.5, ...opts });35 return { adapter, requests, waits };36}37const req = (adapter: ReturnType<typeof createOpenAIAdapter>, over: Partial<ModelRequest> = {}): ModelRequest => ({38 model: 'gpt-6-luna', effort: 'low', system: 'You are a test agent.', tools: [READ, LIST],39 transcript: adapter.initTranscript('hello'), ...over,40});41const result = (callId: string, ok: boolean, content: string): ToolResult => ({ callId, name: 'read_file', ok, content, artifact: null, trust: 'tool' });42const ok = (): Responder => openaiStream({ output: [message('msg_ok', 'ok')] });4344describe('openai request shape', () => {45 it('uses the Responses API statelessly with strict function tools', async () => {46 const { adapter, requests } = setup([ok()]);47 await adapter.call(req(adapter));48 const r = requests[0]!;49 expect(new URL(r.url).pathname).toBe('/v1/responses');50 expect(r.headers.get('authorization')).toBe('Bearer test-key-not-real');51 const b = r.body!;52 expect(b).toMatchObject({53 model: 'gpt-6-luna', store: false, stream: true, instructions: 'You are a test agent.',54 include: ['reasoning.encrypted_content'], reasoning: { effort: 'low', summary: 'auto' },55 max_output_tokens: 64_000, parallel_tool_calls: true,56 });57 for (const forbidden of ['previous_response_id', 'conversation', 'tool_choice', 'temperature', 'top_p', 'truncation', 'background']) expect(b).not.toHaveProperty(forbidden);58 expect(b.tools).toEqual([59 { type: 'function', name: 'list_dir', description: 'List a directory', strict: true, parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'], additionalProperties: false } },60 {61 type: 'function', name: 'read_file', description: 'Read a file', strict: true,62 parameters: { type: 'object', properties: { path: { type: 'string' }, limit: { anyOf: [{ type: 'integer' }, { type: 'null' }] } }, required: ['path', 'limit'], additionalProperties: false },63 },64 ]);65 expect(b.input).toEqual([{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hello' }] }]);66 });6768 it.each(['low', 'medium', 'high', 'xhigh', 'max'] as const)('maps effort %s 1:1', async (effort) => {69 const { adapter, requests } = setup([ok()]);70 await adapter.call(req(adapter, { effort }));71 expect(requests[0]!.body!.reasoning).toEqual({ effort, summary: 'auto' });72 });7374 it('omits effort, instructions and tools when not given, and caps max_output_tokens', async () => {75 const { adapter, requests } = setup([ok()]);76 await adapter.call(req(adapter, { effort: undefined, system: '', tools: [], maxOutputTokens: 1_000_000 }));77 const b = requests[0]!.body!;78 expect(b.reasoning).toEqual({ summary: 'auto' });79 expect(b.max_output_tokens).toBe(128_000);80 for (const absent of ['instructions', 'tools', 'parallel_tool_calls']) expect(b).not.toHaveProperty(absent);81 });82});8384describe('openai streaming and normalisation', () => {85 it('streams text and reasoning-summary deltas and normalises blocks, stop and usage', async () => {86 const { adapter } = setup([openaiStream({87 output: [reasoning('rs_1', ['First thought.', 'Second thought.']), message('msg_1', 'Hello there')],88 usage: { input_tokens: 12_000, output_tokens: 500, cached_tokens: 4000, cache_write_tokens: 7000, reasoning_tokens: 300 },89 })]);90 const deltas: ModelDelta[] = [];91 const res = await adapter.call(req(adapter, { onDelta: (d) => deltas.push(d) }));92 expect(deltas.filter((d) => d.type === 'reasoning').map((d) => d.text).join('')).toBe('First thought.\n\nSecond thought.');93 expect(deltas.filter((d) => d.type === 'text').map((d) => d.text).join('')).toBe('Hello there');94 expect(res.blocks).toEqual([{ type: 'reasoning', summary: 'First thought.\n\nSecond thought.' }, { type: 'text', text: 'Hello there' }]);95 expect(res.stop).toBe('end_turn');96 expect(res.nativeStop).toBe('completed');97 expect(res.model).toBe('gpt-6-luna');98 // input_tokens INCLUDES cached and written tokens: 12000 - 4000 - 7000 = 1000 uncached (the docs' example).99 expect(res.usage).toMatchObject({ inputTokens: 1000, outputTokens: 500, cacheReadTokens: 4000, cacheWriteTokens: 7000, turns: 1 });100 // gpt-6-luna: 0.10 in, 0.50 out, 0.01 cached, 0.125 30-minute cache write per MTok.101 expect(res.usage.usd).toBeCloseTo((1000 * 0.1 + 500 * 0.5 + 4000 * 0.01 + 7000 * 0.125) / 1e6, 12);102 expect(adapter.price('gpt-6-luna', res.usage)).toBe(res.usage.usd);103 // Native: output items verbatim, encrypted reasoning included.104 expect(res.native).toEqual([reasoning('rs_1', ['First thought.', 'Second thought.']), message('msg_1', 'Hello there')]);105 });106107 it('parses parallel function calls and replays reasoning + calls unchanged with the outputs', async () => {108 const firstOutput = [109 reasoning('rs_a', ['Read both files.']),110 call('fc_1', 'call_a', 'read_file', '{"path":"a.txt","limit":null}'),111 call('fc_2', 'call_b', 'read_file', '{"path":"b.txt","limit":20}'),112 ];113 const { adapter, requests } = setup([openaiStream({ output: firstOutput }), ok()]);114 const t0 = adapter.initTranscript('read a.txt and b.txt');115 const first = await adapter.call(req(adapter, { transcript: t0 }));116 expect(first.stop).toBe('tool_use');117 expect(first.blocks).toEqual([118 { type: 'reasoning', summary: 'Read both files.' },119 { type: 'tool_call', id: 'call_a', name: 'read_file', input: { path: 'a.txt' } }, // strict-mode null removed120 { type: 'tool_call', id: 'call_b', name: 'read_file', input: { path: 'b.txt', limit: 20 } },121 ]);122123 const frozen = JSON.stringify(t0);124 const t1 = adapter.appendAssistant(t0, first.native);125 const t2 = adapter.appendToolResults(t1, [result('call_a', true, 'AAA'), result('call_b', false, 'ENOENT: no such file')]);126 expect(JSON.stringify(t0)).toBe(frozen);127 expect(t2.slice(-2)).toEqual([128 { type: 'function_call_output', call_id: 'call_a', output: 'AAA' },129 { type: 'function_call_output', call_id: 'call_b', output: '[tool_error] ENOENT: no such file' },130 ]);131132 await adapter.call(req(adapter, { transcript: t2 }));133 const sent = requests[1]!.body!.input as Record<string, unknown>[];134 expect(JSON.stringify(sent.slice(0, 1))).toBe(JSON.stringify(requests[0]!.body!.input));135 expect(sent.slice(1, 4)).toEqual(firstOutput); // encrypted_content and raw arguments byte for byte136 expect(sent[1]).toMatchObject({ type: 'reasoning', encrypted_content: 'enc-fixture-rs_a' });137 expect(sent).toHaveLength(6);138 // Same request whether the engine spreads `native` or pushes it as one element.139 expect(JSON.stringify(openaiItems([...t0, first.native, ...t2.slice(-2)]))).toBe(JSON.stringify(openaiItems(t2)));140 });141142 it('surfaces invalid JSON arguments and schema mismatches instead of letting the tool run', async () => {143 const { adapter } = setup([openaiStream({144 output: [145 call('fc_1', 'call_bad', 'read_file', '{"path": "a.txt", '),146 call('fc_2', 'call_schema', 'read_file', '{"limit":3}'),147 call('fc_3', 'call_ok', 'list_dir', '{"path":"."}'),148 ],149 })]);150 const res = await adapter.call(req(adapter));151 const [bad, schema, good] = res.blocks.filter((b) => b.type === 'tool_call');152 expect(parseInvalidToolInput(bad!.input)).toEqual({ reason: 'invalid_json', detail: 'arguments are not valid JSON', raw: '{"path": "a.txt", ' });153 expect(parseInvalidToolInput(schema!.input)).toMatchObject({ reason: 'schema', detail: 'path: missing' });154 expect(good!.input).toEqual({ path: '.' });155 expect(invalidToolInputMessage(bad!.input)).toContain('{"INVALID_JSON":"{\\"path\\": \\"a.txt\\", "}');156 expect(res.stop).toBe('tool_use');157 });158159 it('marks a function call cut off by max_output_tokens as truncated and still asks for its result', async () => {160 const { adapter } = setup([openaiStream({161 status: 'incomplete', incomplete_details: { reason: 'max_output_tokens' },162 output: [call('fc_1', 'call_ok', 'list_dir', '{"path":"."}'), call('fc_2', 'call_cut', 'read_file', '{"path":"a-very-lo', 'incomplete')],163 })]);164 const res = await adapter.call(req(adapter));165 expect(res.stop).toBe('tool_use');166 expect(res.nativeStop).toBe('incomplete:max_output_tokens');167 const [good, cut] = res.blocks.filter((b) => b.type === 'tool_call');168 expect(good!.input).toEqual({ path: '.' });169 expect(parseInvalidToolInput(cut!.input)).toMatchObject({ reason: 'truncated', raw: '{"path":"a-very-lo' });170 });171172 it('maps max_output_tokens without tool calls to max_tokens and drops a dangling reasoning item', async () => {173 const { adapter } = setup([openaiStream({174 status: 'incomplete', incomplete_details: { reason: 'max_output_tokens' },175 output: [message('msg_1', 'A long answer that was cut'), reasoning('rs_tail', ['thinking…'])],176 })]);177 const res = await adapter.call(req(adapter));178 expect(res.stop).toBe('max_tokens');179 expect(res.native).toEqual([message('msg_1', 'A long answer that was cut')]);180 expect(res.blocks.map((b) => b.type)).toEqual(['text', 'reasoning']);181 });182183 it('maps a refusal content part, never surfacing that turn\'s tool calls', async () => {184 const refusal = { id: 'msg_r', type: 'message', role: 'assistant', status: 'completed', content: [{ type: 'refusal', refusal: 'I can’t help with that.' }] };185 const { adapter } = setup([openaiStream({ output: [reasoning('rs_1', ['hm']), refusal, call('fc_1', 'call_x', 'list_dir', '{"path":"."}')] })]);186 const res = await adapter.call(req(adapter));187 expect(res.stop).toBe('refusal');188 expect(res.refusal).toEqual({ category: null, explanation: 'I can’t help with that.' });189 expect(res.blocks.some((b) => b.type === 'tool_call')).toBe(false);190 expect(res.native).toEqual([refusal]);191 });192193 it('maps content_filter and policy failures to refusal with the category', async () => {194 const a = setup([openaiStream({ status: 'incomplete', incomplete_details: { reason: 'content_filter' }, output: [message('msg_1', 'partial')] })]);195 const filtered = await a.adapter.call(req(a.adapter));196 expect(filtered.stop).toBe('refusal');197 expect(filtered.refusal).toEqual({ category: 'content_filter', explanation: null });198199 const b = setup([openaiStream({ status: 'failed', error: { code: 'bio_policy', message: 'Blocked by policy.' }, output: [], usage: null })]);200 const failed = await b.adapter.call(req(b.adapter));201 expect(failed.stop).toBe('refusal');202 expect(failed.refusal).toEqual({ category: 'bio_policy', explanation: 'Blocked by policy.' });203 expect(failed.native).toEqual([]);204 expect(failed.usage).toMatchObject({ usd: 0, inputTokens: 0, outputTokens: 0 });205 expect(b.requests).toHaveLength(1);206 });207208 it('falls back to response.output when no output_item.done event was seen', async () => {209 const final = openaiEvents({ output: [message('msg_1', 'hi')] }).filter((e) => e.event === 'response.completed');210 const { adapter } = setup([() => sseResponse(final)]);211 const res = await adapter.call(req(adapter));212 expect(res.blocks).toEqual([{ type: 'text', text: 'hi' }]);213 expect(res.native).toEqual([message('msg_1', 'hi')]);214 });215216 it('prices by the requested ID when the API answers with a dated snapshot, and applies the long-context tier', async () => {217 const { adapter } = setup([openaiStream({ model: 'gpt-6-luna-2026-10-01', output: [message('m', 'x')], usage: { input_tokens: 300_000, output_tokens: 1000 } })]);218 const res = await adapter.call(req(adapter));219 expect(res.model).toBe('gpt-6-luna-2026-10-01');220 // > 272K input tokens: 0.20 in / 0.75 out per MTok for the whole call.221 expect(res.usage.usd).toBeCloseTo((300_000 * 0.2 + 1000 * 0.75) / 1e6, 12);222 });223224 it('normalises usage defensively', () => {225 expect(openaiUsage(null)).toEqual({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 });226 expect(openaiUsage({ input_tokens: 10, output_tokens: 2 })).toEqual({ inputTokens: 10, outputTokens: 2, cacheReadTokens: 0, cacheWriteTokens: 0 });227 });228});229230describe('openai transcript helpers', () => {231 it('appends user messages and developer notes without mutating', () => {232 const { adapter } = setup([]);233 const t0 = adapter.initTranscript('go');234 const t1 = adapter.appendOperatorNote(t0, 'wrap up');235 const t2 = adapter.appendUser(t1, 'steer');236 expect(t0).toHaveLength(1);237 expect(t1[1]).toEqual({ type: 'message', role: 'developer', content: [{ type: 'input_text', text: 'wrap up' }] });238 expect(t2[2]).toEqual({ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'steer' }] });239 });240241 it('rejects a corrupted transcript with a typed error before any request', async () => {242 const { adapter, requests } = setup([]);243 await expect(adapter.call(req(adapter, { transcript: ['not an item'] }))).rejects.toMatchObject({ code: 'invalid_transcript', retryable: false });244 expect(requests).toHaveLength(0);245 });246});247248describe('openai errors, retries and abort', () => {249 it('honours retry-after-ms / retry-after on 429', async () => {250 const { adapter, waits } = setup([251 openaiError(429, 'rate_limit_exceeded', 'slow down', { 'retry-after-ms': '1500' }, 'rate_limit_error'),252 openaiError(429, 'rate_limit_exceeded', 'slow down', { 'retry-after': '3' }, 'rate_limit_error'),253 ok(),254 ]);255 const res = await adapter.call(req(adapter));256 expect(res.attempts).toBe(3);257 expect(waits).toEqual([1500, 3000]);258 });259260 it('does not retry an exhausted quota or client errors', async () => {261 const a = setup([openaiError(429, 'insufficient_quota', 'You exceeded your current quota', {}, 'insufficient_quota')]);262 await expect(a.adapter.call(req(a.adapter))).rejects.toMatchObject({ status: 429, code: 'insufficient_quota', retryable: false, attempts: 1 });263 for (const [status, code] of [[400, 'invalid_request_error'], [401, 'invalid_api_key'], [404, 'model_not_found']] as const) {264 const b = setup([openaiError(status, code, 'nope')]);265 const err = await b.adapter.call(req(b.adapter)).catch((e: unknown) => e);266 expect(err).toBeInstanceOf(ProviderError);267 expect(err).toMatchObject({ provider: 'openai', status, code, retryable: false, requestId: 'req_fixture_oa_err' });268 expect(b.requests).toHaveLength(1);269 }270 });271272 it('retries 5xx with backoff and gives up after maxAttempts', async () => {273 const { adapter, requests, waits } = setup(Array.from({ length: 6 }, () => openaiError(503, null, 'unavailable', {}, 'server_error')));274 await expect(adapter.call(req(adapter))).rejects.toMatchObject({ status: 503, retryable: true, attempts: 6 });275 expect(requests).toHaveLength(6);276 expect(waits).toEqual([750, 1500, 3000, 6000, 12000]);277 });278279 it('retries connection errors, a failed response with server_error and a truncated stream', async () => {280 const truncated: Responder = () => sseResponse(openaiEvents({ output: [message('m', 'partial')] }).slice(0, 3));281 const { adapter, requests } = setup([282 () => { throw new TypeError('fetch failed'); },283 openaiStream({ status: 'failed', error: { code: 'server_error', message: 'The server had an error' }, output: [], usage: null }),284 truncated,285 ok(),286 ]);287 const res = await adapter.call(req(adapter));288 expect(res.attempts).toBe(4);289 expect(requests).toHaveLength(4);290 expect(res.blocks).toEqual([{ type: 'text', text: 'ok' }]);291 });292293 it('retries a transient error event inside the stream but not a permanent one', async () => {294 const errorEvent = (code: string): Responder => () => sseResponse([295 ...openaiEvents({ output: [message('m', 'partial')] }).slice(0, 2),296 { event: 'error', data: { type: 'error', sequence_number: 9, code, message: 'stream error', param: null } },297 ]);298 const a = setup([errorEvent('server_error'), ok()]);299 expect((await a.adapter.call(req(a.adapter))).attempts).toBe(2);300 const b = setup([errorEvent('invalid_prompt'), ok()]);301 await expect(b.adapter.call(req(b.adapter))).rejects.toMatchObject({ retryable: false });302 expect(b.requests).toHaveLength(1);303 });304305 it('aborts through AbortSignal with a non-retryable typed error', async () => {306 const { adapter, requests } = setup([hangUntilAborted, ok()]);307 const controller = new AbortController();308 const pending = adapter.call(req(adapter, { signal: controller.signal }));309 setTimeout(() => controller.abort(), 20);310 await expect(pending).rejects.toMatchObject({ name: 'ProviderError', code: 'aborted', retryable: false });311 expect(requests).toHaveLength(1);312 });313314 it('fails with a typed error when the API key is missing', async () => {315 const saved = process.env.OPENAI_API_KEY;316 delete process.env.OPENAI_API_KEY;317 try {318 const adapter = createOpenAIAdapter({ fetch: fakeFetch([]).fetch });319 await expect(adapter.call(req(adapter))).rejects.toMatchObject({ code: 'missing_api_key', retryable: false });320 expect((await adapter.listModels()).every((m) => !m.available)).toBe(true);321 } finally {322 if (saved !== undefined) process.env.OPENAI_API_KEY = saved;323 }324 });325});326327describe('openai listModels', () => {328 it('marks a catalogue model available only when the live list contains its exact ID', async () => {329 const { adapter, requests } = setup([() => jsonResponse(200, {330 object: 'list',331 data: ['gpt-6-luna', 'gpt-6.1-sol', 'gpt-6-sol', 'gpt-6-astra-2026-09-01'].map((id) => ({ id, object: 'model', created: 1_790_000_000, owned_by: 'openai' })),332 })]);333 const models = await adapter.listModels();334 expect(new URL(requests[0]!.url).pathname).toBe('/v1/models');335 expect(models.map((m) => [m.id, m.tier, m.available])).toEqual([336 ['gpt-6-astra', 'frontier', false], // a dated snapshot is NOT silently substituted337 ['gpt-6.1-sol', 'main', true], ['gpt-6.1-terra', 'worker', false], ['gpt-6-luna', 'volume', true],338 ]);339 });340});341