Repository Public
1import { ModelInfo, ModelResponse } from '@aos/shared';2import { describe, expect, it } from 'vitest';3import { z } from 'zod';4import { CATALOGUE, PRICE_EFFECTIVE_FROM, PRICE_TABLE, getProvider, priceUsage, promptTokens, seedPrices } from './index.js';5import { ProviderError, parseRetryAfter } from './errors.js';6import { backoffMs, withRetries } from './retry.js';7import {8 INVALID_TOOL_INPUT_PREFIX, invalidToolInputMessage, markInvalidToolInput, parseInvalidToolInput,9 stripNullOptionals, toStrictSchema, validateToolInput,10} from './schema.js';11import { anthropicStream, fakeFetch } from './test/fixtures.js';12import { createAnthropicAdapter } from './anthropic.js';1314describe('catalogue', () => {15 it('lists exactly the eight contract models with their tiers, unavailable until checked live', () => {16 expect(CATALOGUE.map((m) => `${m.provider}:${m.id}:${m.tier}`)).toEqual([17 'anthropic:claude-fable-5-2:frontier', 'anthropic:claude-opus-5-5:main', 'anthropic:claude-sonnet-5-5:worker', 'anthropic:claude-haiku-5-5:volume',18 'openai:gpt-6-astra:frontier', 'openai:gpt-6.1-sol:main', 'openai:gpt-6.1-terra:worker', 'openai:gpt-6-luna:volume',19 ]);20 for (const m of CATALOGUE) {21 expect(ModelInfo.parse(m)).toEqual(m);22 expect(m.available).toBe(false);23 expect(m.efforts).toEqual(['low', 'medium', 'high', 'xhigh', 'max']);24 }25 });2627 it('has a price row for every catalogue model, matching the catalogue', () => {28 for (const m of CATALOGUE) {29 const row = PRICE_TABLE.find((r) => r.provider === m.provider && r.model === m.id);30 expect(row, m.id).toBeDefined();31 expect(row).toMatchObject({ inputUsd: m.inputUsd, outputUsd: m.outputUsd, effectiveFrom: '2026-11-10' });32 }33 });34});3536describe('price()', () => {37 const u = (inputTokens: number, outputTokens: number, cacheReadTokens = 0, cacheWriteTokens = 0) => ({ inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens });3839 it.each([40 ['anthropic', 'claude-fable-5-2', 8, 40, 0.2, 10],41 ['anthropic', 'claude-opus-5-5', 4, 20, 0.2, 5],42 ['anthropic', 'claude-sonnet-5-5', 2, 10, 0.1, 2.5],43 ['anthropic', 'claude-haiku-5-5', 0.1, 0.5, 0.01, 0.125],44 ['openai', 'gpt-6-astra', 10, 50, 1, 12.5],45 ['openai', 'gpt-6.1-sol', 2, 10, 0.1, 2.5],46 ['openai', 'gpt-6.1-terra', 1, 5, 0.05, 1.25],47 ['openai', 'gpt-6-luna', 0.1, 0.5, 0.01, 0.125],48 ] as const)('%s/%s: %d in, %d out, %d cache read, %d cache write per MTok', (provider, model, i, o, cr, cw) => {49 // 90K-token prompts stay below every long-prompt threshold.50 expect(priceUsage(provider, model, u(90_000, 0))).toBeCloseTo(i * 0.09, 12);51 expect(priceUsage(provider, model, u(0, 1_000_000))).toBe(o);52 // Cache tokens are priced on their own, below the long-prompt thresholds.53 expect(priceUsage(provider, model, u(0, 0, 90_000, 0))).toBeCloseTo(cr * 0.09, 12);54 expect(priceUsage(provider, model, u(0, 0, 0, 90_000))).toBeCloseTo(cw * 0.09, 12);55 expect(getProvider(provider).price(model, u(1234, 567, 89, 10))).toBeCloseTo((1234 * i + 567 * o + 89 * cr + 10 * cw) / 1e6, 12);56 });5758 it('applies the long-prompt tier to the whole call (prompt = uncached + cache read + cache write)', () => {59 expect(priceUsage('openai', 'gpt-6-astra', u(272_000, 1000))).toBeCloseTo((272_000 * 10 + 1000 * 50) / 1e6, 12);60 expect(priceUsage('openai', 'gpt-6-astra', u(100_000, 1000, 172_001))).toBeCloseTo((100_000 * 20 + 1000 * 75 + 172_001 * 2) / 1e6, 12);61 expect(priceUsage('anthropic', 'claude-haiku-5-5', u(100_001, 1000))).toBeCloseTo((100_001 * 0.5 + 1000 * 2.5) / 1e6, 12);62 expect(promptTokens({ inputTokens: 1, cacheReadTokens: 2, cacheWriteTokens: 3 })).toBe(6);63 });6465 it('prices a dated snapshot ID as its base model', () => {66 expect(priceUsage('openai', 'gpt-6-luna-2026-10-01', u(90_000, 0))).toBeCloseTo(0.009, 12);67 expect(priceUsage('anthropic', 'claude-opus-5-5-20260901', u(0, 1000))).toBeCloseTo(0.02, 12);68 });6970 it('refuses to price an unknown model at zero', () => {71 expect(() => priceUsage('openai', 'gpt-9', u(1, 1))).toThrowError(ProviderError);72 expect(() => getProvider('anthropic').price('claude-unknown', u(1, 1))).toThrow(/No price/);73 });74});7576describe('seedPrices()', () => {77 it('upserts every row with the effective date 2026-11-10 and is idempotent', async () => {78 // In-memory stand-in for model_prices keyed by its primary key (provider, model, effective_from).79 const table = new Map<string, unknown[]>();80 const statements: string[] = [];81 const db = {82 async query(sql: string, params: unknown[] = []) {83 statements.push(sql);84 table.set(params.slice(0, 3).join('|'), params);85 },86 };87 await seedPrices(db);88 await seedPrices(db);89 expect(table.size).toBe(PRICE_TABLE.length);90 expect(statements).toHaveLength(PRICE_TABLE.length * 2);91 expect(statements[0]).toMatch(/INSERT INTO model_prices \(provider, model, effective_from, input_usd, output_usd, cache_read_usd, cache_write_usd\)/);92 expect(statements[0]).toMatch(/ON CONFLICT \(provider, model, effective_from\) DO UPDATE/);93 expect(table.get(`anthropic|claude-opus-5-5|${PRICE_EFFECTIVE_FROM}`)).toEqual(['anthropic', 'claude-opus-5-5', '2026-11-10', 4, 20, 0.2, 5]);94 expect(table.get('openai|gpt-6-luna|2026-11-10')).toEqual(['openai', 'gpt-6-luna', '2026-11-10', 0.1, 0.5, 0.01, 0.125]);95 for (const m of CATALOGUE) expect(table.has(`${m.provider}|${m.id}|2026-11-10`)).toBe(true);96 });97});9899describe('toStrictSchema()', () => {100 const Input = z.object({101 path: z.string().min(1).describe('File path'),102 limit: z.number().int().min(1).max(5000).optional(),103 mode: z.enum(['text', 'binary']).default('text'),104 range: z.object({ from: z.number().int(), to: z.number().int().optional() }).optional(),105 tags: z.array(z.string().max(20)).max(10),106 note: z.string().nullable(),107 });108 const original = z.toJSONSchema(Input, { io: 'input' }) as Record<string, unknown>;109 const strict = toStrictSchema(original);110111 it('converts a zod-derived schema into the strict subset', () => {112 expect(strict).toEqual({113 type: 'object',114 properties: {115 path: { type: 'string', description: 'File path' },116 limit: { anyOf: [{ type: 'integer' }, { type: 'null' }] },117 mode: { anyOf: [{ type: 'string', enum: ['text', 'binary'] }, { type: 'null' }] },118 range: {119 anyOf: [{120 type: 'object',121 properties: { from: { type: 'integer' }, to: { anyOf: [{ type: 'integer' }, { type: 'null' }] } },122 required: ['from', 'to'], additionalProperties: false,123 }, { type: 'null' }],124 },125 tags: { type: 'array', items: { type: 'string' } },126 note: { anyOf: [{ type: 'string' }, { type: 'null' }] },127 },128 required: ['path', 'limit', 'mode', 'range', 'tags', 'note'],129 additionalProperties: false,130 });131 });132133 it('is idempotent and deterministic', () => {134 expect(toStrictSchema(strict)).toEqual(strict);135 expect(JSON.stringify(toStrictSchema(original))).toBe(JSON.stringify(strict));136 });137138 it('handles $defs/$ref, oneOf and non-object roots', () => {139 expect(toStrictSchema({140 type: 'object', properties: { a: { $ref: '#/$defs/A' }, b: { oneOf: [{ type: 'string', format: 'regex' }, { type: 'number', maximum: 3 }] } },141 required: ['a', 'b'], $defs: { A: { type: 'object', properties: { x: { type: 'string', format: 'uuid' } } } },142 })).toEqual({143 type: 'object', properties: { a: { $ref: '#/$defs/A' }, b: { anyOf: [{ type: 'string' }, { type: 'number' }] } },144 required: ['a', 'b'], additionalProperties: false,145 $defs: { A: { type: 'object', properties: { x: { anyOf: [{ type: 'string', format: 'uuid' }, { type: 'null' }] } }, required: ['x'], additionalProperties: false } },146 });147 expect(toStrictSchema({ type: 'string' })).toEqual({ type: 'object', properties: {}, required: [], additionalProperties: false });148 });149150 it('round-trips model output back to what zod expects', () => {151 const fromModel = { path: 'a.txt', limit: null, mode: null, range: { from: 1, to: null }, tags: ['x'], note: null };152 expect(validateToolInput(strict, fromModel)).toBeNull();153 const cleaned = stripNullOptionals(original, fromModel);154 // Optional fields lose their null; a genuinely nullable field keeps it.155 expect(cleaned).toEqual({ path: 'a.txt', range: { from: 1 }, tags: ['x'], note: null });156 expect(Input.parse(cleaned)).toEqual({ path: 'a.txt', mode: 'text', range: { from: 1 }, tags: ['x'], note: null });157 });158159 it('validates structure leniently', () => {160 expect(validateToolInput(strict, { path: 'a', limit: null, mode: 'text', range: null, tags: [], note: null })).toBeNull();161 expect(validateToolInput(strict, { path: 'a' })).toBe('limit: missing');162 expect(validateToolInput(strict, { path: 'a', limit: 1.5, mode: null, range: null, tags: [], note: null })).toMatch(/^limit: /);163 expect(validateToolInput(strict, { path: 'a', limit: null, mode: 'hex', range: null, tags: [], note: null })).toMatch(/^mode: /);164 expect(validateToolInput(strict, { path: 'a', limit: null, mode: null, range: null, tags: [1], note: null })).toBe('tags[0]: expected string, got integer');165 expect(validateToolInput(strict, { path: 'a', limit: null, mode: null, range: null, tags: [], note: null, extra: 1 })).toBe('extra: unknown property');166 expect(validateToolInput(strict, 'nope')).toMatch(/expected object/);167 expect(validateToolInput({ type: 'object', properties: { a: { $ref: '#/nowhere' } } }, { a: 1 })).toBeNull();168 });169});170171describe('invalid tool input markers', () => {172 it('is a string that no object schema accepts, and round-trips', () => {173 const marker = markInvalidToolInput({ reason: 'invalid_json', detail: 'bad', raw: '{"a": "b' });174 expect(marker.startsWith(INVALID_TOOL_INPUT_PREFIX)).toBe(true);175 expect(z.object({}).safeParse(marker).success).toBe(false);176 expect(z.object({ a: z.string().optional() }).safeParse(marker).success).toBe(false);177 expect(parseInvalidToolInput(marker)).toEqual({ reason: 'invalid_json', detail: 'bad', raw: '{"a": "b' });178 expect(invalidToolInputMessage(marker)).toContain(JSON.stringify({ INVALID_JSON: '{"a": "b' }));179 expect(parseInvalidToolInput({ a: 1 })).toBeNull();180 expect(parseInvalidToolInput('plain string')).toBeNull();181 expect(markInvalidToolInput({ reason: 'truncated', detail: '', raw: 'x'.repeat(10_000) }).length).toBeLessThan(4200);182 });183});184185describe('retry helpers', () => {186 it('parses retry-after headers', () => {187 expect(parseRetryAfter(new Headers({ 'retry-after': '7' }))).toBe(7000);188 expect(parseRetryAfter(new Headers({ 'retry-after': '7', 'retry-after-ms': '250' }))).toBe(250);189 expect(parseRetryAfter(new Headers({ 'retry-after': 'Wed, 11 Nov 2026 00:00:10 GMT' }), Date.parse('2026-11-11T00:00:00Z'))).toBe(10_000);190 expect(parseRetryAfter(new Headers({ 'retry-after': 'soon' }))).toBeNull();191 expect(parseRetryAfter(new Headers())).toBeNull();192 expect(parseRetryAfter(undefined)).toBeNull();193 });194195 it('backs off exponentially with jitter and a 30 s ceiling', () => {196 expect([1, 2, 3, 4, 5, 6, 10].map((n) => backoffMs(n, () => 1))).toEqual([1000, 2000, 4000, 8000, 16000, 30000, 30000]);197 expect(backoffMs(3, () => 0)).toBe(2000);198 });199200 it('caps a very long retry-after and never retries foreign errors', async () => {201 const waits: number[] = [];202 let n = 0;203 const out = await withRetries(async () => {204 if (n++ === 0) throw new ProviderError('busy', { provider: 'openai', retryable: true, retryAfterMs: 3_600_000 });205 return 'ok';206 }, { maxAttempts: 6, sleep: async (ms) => { waits.push(ms); }, random: () => 0.5 }, () => new ProviderError('aborted', { provider: 'openai', retryable: false }));207 expect(out).toEqual({ value: 'ok', attempts: 2 });208 expect(waits).toEqual([120_000]);209 await expect(withRetries(async () => { throw new TypeError('bug'); }, { maxAttempts: 6, sleep: async () => {}, random: () => 0 }, () => new ProviderError('x', { provider: 'openai', retryable: false }))).rejects.toBeInstanceOf(TypeError);210 });211});212213describe('contract surface', () => {214 it('getProvider returns one adapter per provider with the contract methods', () => {215 for (const id of ['anthropic', 'openai'] as const) {216 const a = getProvider(id);217 expect(a.id).toBe(id);218 expect(getProvider(id)).toBe(a);219 for (const m of ['initTranscript', 'appendToolResults', 'appendUser', 'appendOperatorNote', 'call', 'listModels', 'price'] as const) expect(typeof a[m]).toBe('function');220 }221 });222223 it('returns a response that satisfies the frozen ModelResponse schema and survives a JSON round-trip (step journal)', async () => {224 const adapter = createAnthropicAdapter({225 apiKey: 'test-key-not-real',226 fetch: fakeFetch([anthropicStream({ content: [{ type: 'thinking', thinking: 't', signature: 's' }, { type: 'tool_use', id: 'toolu_1', name: 'x', json: '{}' }], stop_reason: 'tool_use' })]).fetch,227 });228 const res = await adapter.call({ model: 'claude-haiku-5-5', system: 's', tools: [], transcript: adapter.initTranscript('hi') });229 const journaled = JSON.parse(JSON.stringify(res)) as unknown;230 expect(journaled).toEqual(res);231 expect(ModelResponse.parse(journaled)).toMatchObject({ stop: 'tool_use', model: 'claude-haiku-5-5' });232 });233});234