Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""OpenAI core live probes (Responses / Chat Completions / Completions / Conversations).34Cheapest models, "Reply with OK.", max_output_tokens <= 32 (one documented fallback), every call logged by5scripts.live. Sanitized raw responses -> tmp-live/openai-core/*.json. Prints a summary table + JSON summary.6"""7from __future__ import annotations8import json, sys, time, traceback9from pathlib import Path1011ROOT = Path('/Users/simon-pierreboucher/Desktop/doc-api')12sys.path.insert(0, str(ROOT))13from scripts import live # noqa: E4021415OUT = ROOT / 'tmp-live' / 'openai-core'16OUT.mkdir(parents=True, exist_ok=True)17R = 'gpt-5.4-nano' # Responses reasoning model: $0.20/M in, $0.02/M cached, $1.25/M out18C = 'gpt-4.1-nano' # Chat model: $0.10/M in, $0.025/M cached, $0.40/M out19L = 'gpt-3.5-turbo-instruct' # legacy: $1.50/M in, $2.00/M out20PRICE = {R: (0.20, 0.02, 1.25), C: (0.10, 0.025, 0.40), L: (1.50, 1.50, 2.00)}21PNG_1x1 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='2223summary: list[dict] = []24total_cost = 0.0252627def cost_from_usage(model: str, usage: dict | None) -> float:28 if not usage:29 return 0.030 pin, pcached, pout = PRICE.get(model, (1.0, 1.0, 1.0))31 it = usage.get('input_tokens', usage.get('prompt_tokens', 0)) or 032 ot = usage.get('output_tokens', usage.get('completion_tokens', 0)) or 033 cached = (usage.get('input_tokens_details') or usage.get('prompt_tokens_details') or {}).get('cached_tokens', 0) or 034 return ((it - cached) * pin + cached * pcached + ot * pout) / 1e6353637def save(name: str, obj) -> None:38 live.save_sanitized(obj, OUT / f'{name}.json')394041def record(name: str, status: str, http: int | None, note: str = '', cost: float = 0.0) -> None:42 global total_cost43 total_cost += cost44 summary.append({'test': name, 'status': status, 'http': http, 'note': note[:400], 'est_cost_usd': round(cost, 6)})45 print(f'[{status}] {name} http={http} cost=${cost:.6f} {note[:200]}', flush=True)464748def call(name: str, method: str, path: str, body=None, model: str | None = None, est: float = 0.0002, **kw):49 st, out, hdrs = live.openai_request(method, path, body, est_cost_usd=est, note=f'openai-core {name}', **kw)50 save(name, {'request': body, 'http_status': st, 'headers': live.interesting_headers(hdrs), 'body': out})51 return st, out, hdrs525354def parse_sse(lines):55 """Yield (event_name, data_str) from an SSE line iterator."""56 ev, data = None, []57 for line in lines:58 if line == '':59 if data:60 yield ev, '\n'.join(data)61 ev, data = None, []62 elif line.startswith('event:'):63 ev = line[6:].strip()64 elif line.startswith('data:'):65 data.append(line[5:].strip())66 if data:67 yield ev, '\n'.join(data)686970def stream_call(name: str, path: str, body=None, method: str = 'POST', est: float = 0.0003):71 st, gen, hdrs = live.openai_request(method, path, body, stream=True, est_cost_usd=est, note=f'openai-core {name}')72 events = []73 if st != 200:74 save(name, {'request': body, 'http_status': st, 'body': gen if not hasattr(gen, '__next__') else None})75 return st, [], hdrs76 for ev, data in parse_sse(gen):77 try:78 payload = json.loads(data)79 except Exception:80 payload = data81 events.append({'event': ev, 'data': payload})82 save(name, {'request': body, 'http_status': st, 'headers': live.interesting_headers(hdrs), 'events': events})83 return st, events, hdrs848586def run(name, fn):87 try:88 fn()89 except Exception as e: # noqa: BLE00190 traceback.print_exc()91 record(name, 'FAILED_VERIFICATION', None, f'exception: {type(e).__name__}: {e}')929394state: dict = {}9596# ---------------------------------------------------------------- (a) minimal97def t_a():98 body = {'model': R, 'input': 'Reply with OK.', 'max_output_tokens': 32}99 st, out, _ = call('a-responses-minimal', 'POST', '/v1/responses', body, R)100 if st == 200:101 state['a_id'] = out['id']102 record('a-responses-minimal', 'LIVE_VERIFIED', st, f"status={out.get('status')} text={[c.get('text') for i in out['output'] if i['type']=='message' for c in i['content']]} keys={sorted(out.keys())}", cost_from_usage(R, out.get('usage')))103 else:104 record('a-responses-minimal', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])105106# ---------------------------------------------------------------- (a2) beta=true variant107def t_a2():108 body = {'model': R, 'input': 'Reply with OK.', 'max_output_tokens': 32}109 st, out, _ = call('a2-responses-beta-query', 'POST', '/v1/responses?beta=true', body, R)110 if st == 200:111 record('a2-responses-beta-query', 'LIVE_VERIFIED', st, f"status={out.get('status')} extra_keys={sorted(set(out.keys()) - set(state.get('a_keys', [])))}", cost_from_usage(R, out.get('usage')))112 else:113 record('a2-responses-beta-query', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])114115# ---------------------------------------------------------------- (b) stream116def t_b():117 body = {'model': R, 'input': 'Reply with OK.', 'max_output_tokens': 32, 'stream': True}118 st, events, _ = stream_call('b-responses-stream', '/v1/responses', body)119 types = [e['data'].get('type') if isinstance(e['data'], dict) else e['data'] for e in events]120 state['b_types'] = types121 usage = None122 for e in events:123 if isinstance(e['data'], dict) and e['data'].get('type') == 'response.completed':124 usage = e['data']['response'].get('usage')125 record('b-responses-stream', 'LIVE_VERIFIED' if st == 200 and types else 'FAILED_VERIFICATION', st, 'events=' + ','.join(map(str, types)), cost_from_usage(R, usage))126127# ---------------------------------------------------------------- (c) previous_response_id128def t_c():129 body = {'model': R, 'input': 'Reply with OK again.', 'previous_response_id': state['a_id'], 'max_output_tokens': 32}130 st, out, _ = call('c-responses-previous-response-id', 'POST', '/v1/responses', body, R)131 if st == 200:132 state['c_id'] = out['id']133 record('c-responses-previous-response-id', 'LIVE_VERIFIED', st, f"previous_response_id={out.get('previous_response_id')} input_tokens={out['usage']['input_tokens']}", cost_from_usage(R, out.get('usage')))134 else:135 record('c-responses-previous-response-id', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])136137# ---------------------------------------------------------------- (d) store=false138def t_d():139 body = {'model': R, 'input': 'Reply with OK.', 'store': False, 'max_output_tokens': 32, 'reasoning': {'effort': 'low'}}140 st, out, _ = call('d-responses-store-false', 'POST', '/v1/responses', body, R)141 if st == 200:142 state['d_id'] = out['id']143 enc = [bool(i.get('encrypted_content')) for i in out['output'] if i['type'] == 'reasoning']144 record('d-responses-store-false', 'LIVE_VERIFIED', st, f"store={out.get('store')} status={out.get('status')} reasoning_items_with_encrypted_content={enc} output_types={[i['type'] for i in out['output']]}", cost_from_usage(R, out.get('usage')))145 # follow-up: retrieving a store=false response should 404146 st2, out2, _ = call('d2-retrieve-unstored', 'GET', f"/v1/responses/{out['id']}", None, R, est=0)147 record('d2-retrieve-unstored', 'LIVE_VERIFIED' if st2 == 404 else 'LIVE_DISCOVERED', st2, json.dumps(out2)[:200])148 else:149 record('d-responses-store-false', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])150151# ---------------------------------------------------------------- (e) structured output152def t_e():153 schema = {'type': 'object', 'properties': {'answer': {'type': 'string'}}, 'required': ['answer'], 'additionalProperties': False}154 body = {'model': R, 'input': 'Reply with OK.', 'max_output_tokens': 32,155 'text': {'format': {'type': 'json_schema', 'name': 'ok_reply', 'strict': True, 'schema': schema}}}156 st, out, _ = call('e-responses-structured-output', 'POST', '/v1/responses', body, R)157 if st == 200:158 texts = [c.get('text') for i in out['output'] if i['type'] == 'message' for c in i['content']]159 parsed = None160 try:161 parsed = json.loads(texts[0]) if texts else None162 except Exception as ex: # noqa: BLE001163 parsed = f'unparseable: {ex}'164 record('e-responses-structured-output', 'LIVE_VERIFIED', st, f"text.format echoed={out.get('text')} output={texts} parsed={parsed} status={out.get('status')}", cost_from_usage(R, out.get('usage')))165 else:166 record('e-responses-structured-output', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])167168# ---------------------------------------------------------------- (f) reasoning effort low + summary auto169def t_f():170 body = {'model': R, 'input': 'What is 2+2? Reply with just the number.', 'max_output_tokens': 32,171 'reasoning': {'effort': 'low', 'summary': 'auto'}}172 st, out, _ = call('f-responses-reasoning-summary', 'POST', '/v1/responses', body, R)173 if st == 200 and out.get('status') == 'incomplete':174 # Documented fallback: reasoning consumed the 32-token budget -> retry with 256 (cost ~ $0.0003)175 body['max_output_tokens'] = 256176 st, out, _ = call('f-responses-reasoning-summary-retry256', 'POST', '/v1/responses', body, R, est=0.0005)177 if st == 200:178 rs = [i for i in out['output'] if i['type'] == 'reasoning']179 record('f-responses-reasoning-summary', 'LIVE_VERIFIED', st,180 f"status={out.get('status')} incomplete={out.get('incomplete_details')} reasoning_echo={out.get('reasoning')} reasoning_tokens={out['usage']['output_tokens_details'].get('reasoning_tokens')} summaries={[s for r in rs for s in r.get('summary', [])]} max_output_tokens={body['max_output_tokens']}",181 cost_from_usage(R, out.get('usage')))182 else:183 record('f-responses-reasoning-summary', 'FAILED_VERIFICATION', st, json.dumps(out)[:300])184185# ---------------------------------------------------------------- (g) retrieve / input_items / delete186def t_g():187 rid = state['a_id']188 st, out, _ = call('g1-responses-retrieve', 'GET', f'/v1/responses/{rid}', None, R, est=0)189 record('g1-responses-retrieve', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"status={out.get('status') if isinstance(out, dict) else out}")190 st, out, _ = call('g2-responses-input-items', 'GET', f'/v1/responses/{state.get("c_id", rid)}/input_items?limit=5', None, R, est=0)191 record('g2-responses-input-items', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"object={out.get('object')} n={len(out.get('data', []))} types={[i.get('type') for i in out.get('data', [])]} has_more={out.get('has_more')}" if isinstance(out, dict) else str(out))192 st, out, _ = call('g3-responses-retrieve-include-logprobs', 'GET', f'/v1/responses/{rid}?include[]=message.output_text.logprobs', None, R, est=0)193 record('g3-responses-retrieve-include', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200] if st != 200 else 'ok')194195def t_g_delete():196 # delete the chain child first, then the parent197 for key, name in (('c_id', 'g4-responses-delete-child'), ('a_id', 'g5-responses-delete-parent')):198 if key in state:199 st, out, _ = call(name, 'DELETE', f'/v1/responses/{state[key]}', None, R, est=0)200 record(name, 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])201 st, out, _ = call('g6-responses-retrieve-deleted', 'GET', f"/v1/responses/{state['a_id']}", None, R, est=0)202 record('g6-responses-retrieve-deleted', 'LIVE_VERIFIED' if st == 404 else 'LIVE_DISCOVERED', st, json.dumps(out)[:200])203204# ---------------------------------------------------------------- (h) input tokens205def t_h():206 body = {'model': R, 'input': 'Reply with OK.'}207 st, out, _ = call('h-responses-input-tokens', 'POST', '/v1/responses/input_tokens', body, R, est=0)208 record('h-responses-input-tokens', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:300])209 body2 = {'model': R, 'instructions': 'You are terse.', 'input': [{'role': 'user', 'content': [{'type': 'input_text', 'text': 'Reply with OK.'}, {'type': 'input_image', 'image_url': PNG_1x1, 'detail': 'low'}]}],210 'tools': [{'type': 'function', 'name': 'noop', 'description': 'Does nothing', 'parameters': {'type': 'object', 'properties': {}, 'additionalProperties': False}}]}211 st, out, _ = call('h2-responses-input-tokens-rich', 'POST', '/v1/responses/input_tokens', body2, R, est=0)212 record('h2-responses-input-tokens-rich', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:300])213 st, out, _ = call('h3-responses-input-tokens-beta', 'POST', '/v1/responses/input_tokens?beta=true', body, R, est=0)214 record('h3-responses-input-tokens-beta', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])215216# ---------------------------------------------------------------- (i) compact217def t_i():218 body = {'model': R, 'input': [{'role': 'user', 'content': 'Reply with OK.'}, {'role': 'assistant', 'content': 'OK'}, {'role': 'user', 'content': 'Reply with OK again.'}]}219 st, out, _ = call('i-responses-compact', 'POST', '/v1/responses/compact', body, R, est=0.0005)220 if st == 200:221 record('i-responses-compact', 'LIVE_VERIFIED', st, f"object={out.get('object')} keys={sorted(out.keys())} output_types={[i.get('type') for i in out.get('output', [])]} usage={out.get('usage')}", cost_from_usage(R, out.get('usage')))222 else:223 record('i-responses-compact', 'FAILED_VERIFICATION', st, json.dumps(out)[:400])224 if 'c_id' in state:225 body2 = {'model': R, 'previous_response_id': state['c_id']}226 st, out, _ = call('i2-responses-compact-previous-response-id', 'POST', '/v1/responses/compact', body2, R, est=0.0005)227 record('i2-responses-compact-previous-response-id', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, (f"output_types={[i.get('type') for i in out.get('output', [])]} usage={out.get('usage')}" if st == 200 else json.dumps(out)[:400]), cost_from_usage(R, out.get('usage')) if st == 200 else 0)228229# ---------------------------------------------------------------- (j) background + cancel, background+stream resume230def t_j():231 body = {'model': R, 'input': 'Reply with OK.', 'background': True, 'max_output_tokens': 32}232 st, out, _ = call('j1-responses-background-create', 'POST', '/v1/responses', body, R)233 if st != 200:234 record('j1-responses-background-create', 'FAILED_VERIFICATION', st, json.dumps(out)[:300]); return235 rid = out['id']236 record('j1-responses-background-create', 'LIVE_VERIFIED', st, f"status={out.get('status')} background={out.get('background')} keys={sorted(out.keys())}")237 st, out, _ = call('j2-responses-cancel-immediately', 'POST', f'/v1/responses/{rid}/cancel', None, R, est=0)238 record('j2-responses-cancel-immediately', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"status={out.get('status') if isinstance(out, dict) else ''} {json.dumps(out)[:200] if st != 200 else ''}")239 final = None240 for _ in range(10):241 st, out, _ = call('j3-responses-background-poll', 'GET', f'/v1/responses/{rid}', None, R, est=0)242 final = out.get('status') if isinstance(out, dict) else None243 if final not in ('queued', 'in_progress'):244 break245 time.sleep(1)246 record('j3-responses-background-poll', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"final_status={final} usage={out.get('usage') if isinstance(out, dict) else None}", cost_from_usage(R, out.get('usage')) if isinstance(out, dict) else 0)247 st, out, _ = call('j4-responses-cancel-again', 'POST', f'/v1/responses/{rid}/cancel', None, R, est=0)248 record('j4-responses-cancel-again-idempotent', 'LIVE_VERIFIED' if st == 200 else 'LIVE_DISCOVERED', st, f"status={out.get('status') if isinstance(out, dict) else ''} {json.dumps(out)[:200]}")249 # background + stream, then resume via GET ?stream=true&starting_after250 body = {'model': R, 'input': 'Reply with OK.', 'background': True, 'stream': True, 'max_output_tokens': 32}251 st, events, _ = stream_call('j5-responses-background-stream', '/v1/responses', body)252 types = [e['data'].get('type') if isinstance(e['data'], dict) else e['data'] for e in events]253 seqs = [e['data'].get('sequence_number') for e in events if isinstance(e['data'], dict)]254 bid = next((e['data']['response']['id'] for e in events if isinstance(e['data'], dict) and 'response' in e['data']), None)255 usage = next((e['data']['response'].get('usage') for e in events if isinstance(e['data'], dict) and e['data'].get('type') == 'response.completed'), None)256 record('j5-responses-background-stream', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"events={types} seqs={seqs[:3]}..{seqs[-1:] if seqs else None}", cost_from_usage(R, usage))257 if bid and len(seqs) > 2:258 st, events2, _ = stream_call('j6-responses-stream-resume', f'/v1/responses/{bid}?stream=true&starting_after={seqs[1]}', None, method='GET', est=0)259 types2 = [e['data'].get('type') if isinstance(e['data'], dict) else e['data'] for e in events2]260 record('j6-responses-stream-resume', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"resumed_after={seqs[1]} events={types2}")261 call('j7-responses-delete-background', 'DELETE', f'/v1/responses/{bid}', None, R, est=0)262 call('j8-responses-delete-cancelled', 'DELETE', f'/v1/responses/{rid}', None, R, est=0)263264# ---------------------------------------------------------------- (k) conversations265def t_k():266 st, out, _ = call('k1-conversations-create', 'POST', '/v1/conversations', {'metadata': {'atlas': 'openai-core'}}, R, est=0)267 if st != 200:268 record('k1-conversations-create', 'FAILED_VERIFICATION', st, json.dumps(out)[:300]); return269 cid = out['id']270 record('k1-conversations-create', 'LIVE_VERIFIED', st, f"object={out.get('object')} keys={sorted(out.keys())}")271 st, out, _ = call('k2-conversations-items-create', 'POST', f'/v1/conversations/{cid}/items', {'items': [{'type': 'message', 'role': 'user', 'content': [{'type': 'input_text', 'text': 'Reply with OK.'}]}]}, R, est=0)272 record('k2-conversations-items-create', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"object={out.get('object')} n={len(out.get('data', []))} item_keys={sorted(out['data'][0].keys()) if out.get('data') else None}" if isinstance(out, dict) else str(out))273 item_id = out['data'][0]['id'] if st == 200 and out.get('data') else None274 st, out, _ = call('k3-conversations-items-list', 'GET', f'/v1/conversations/{cid}/items?limit=10&order=asc', None, R, est=0)275 record('k3-conversations-items-list', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"n={len(out.get('data', []))} first_id={out.get('first_id')} has_more={out.get('has_more')}" if isinstance(out, dict) else str(out))276 if item_id:277 st, out, _ = call('k4-conversations-item-retrieve', 'GET', f'/v1/conversations/{cid}/items/{item_id}', None, R, est=0)278 record('k4-conversations-item-retrieve', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])279 body = {'model': R, 'conversation': cid, 'input': 'Reply with OK once more.', 'max_output_tokens': 32}280 st, out, _ = call('k5-responses-with-conversation', 'POST', '/v1/responses', body, R)281 record('k5-responses-with-conversation', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"conversation={out.get('conversation')} input_tokens={out.get('usage', {}).get('input_tokens')} status={out.get('status')}" if st == 200 else json.dumps(out)[:300], cost_from_usage(R, out.get('usage')) if st == 200 else 0)282 st, out, _ = call('k6-conversations-items-list-after-response', 'GET', f'/v1/conversations/{cid}/items?order=asc', None, R, est=0)283 record('k6-conversations-items-list-after-response', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"types={[i.get('type') for i in out.get('data', [])]}" if isinstance(out, dict) else str(out))284 st, out, _ = call('k7-conversations-update', 'POST', f'/v1/conversations/{cid}', {'metadata': {'atlas': 'openai-core', 'updated': 'yes'}}, R, est=0)285 record('k7-conversations-update', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"metadata={out.get('metadata')}" if isinstance(out, dict) else str(out))286 st, out, _ = call('k8-conversations-retrieve', 'GET', f'/v1/conversations/{cid}', None, R, est=0)287 record('k8-conversations-retrieve', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])288 if item_id:289 st, out, _ = call('k9-conversations-item-delete', 'DELETE', f'/v1/conversations/{cid}/items/{item_id}', None, R, est=0)290 record('k9-conversations-item-delete', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"returns object={out.get('object')} keys={sorted(out.keys()) if isinstance(out, dict) else None}")291 st, out, _ = call('k10-conversations-delete', 'DELETE', f'/v1/conversations/{cid}', None, R, est=0)292 record('k10-conversations-delete', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])293 st, out, _ = call('k11-conversations-retrieve-deleted', 'GET', f'/v1/conversations/{cid}', None, R, est=0)294 record('k11-conversations-retrieve-deleted', 'LIVE_VERIFIED' if st == 404 else 'LIVE_DISCOVERED', st, json.dumps(out)[:200])295296# ---------------------------------------------------------------- (l) chat completions297def t_l():298 msgs = [{'role': 'user', 'content': 'Reply with OK.'}]299 body = {'model': C, 'messages': msgs, 'max_completion_tokens': 32}300 st, out, _ = call('l1-chat-minimal', 'POST', '/v1/chat/completions', body, C)301 record('l1-chat-minimal', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"content={out['choices'][0]['message'].get('content')!r} finish={out['choices'][0].get('finish_reason')} keys={sorted(out.keys())} usage={out.get('usage')}" if st == 200 else json.dumps(out)[:300], cost_from_usage(C, out.get('usage')) if st == 200 else 0)302 body = {'model': C, 'messages': msgs, 'max_completion_tokens': 32, 'stream': True, 'stream_options': {'include_usage': True}}303 st, events, _ = stream_call('l2-chat-stream', '/v1/chat/completions', body)304 shapes = []305 usage = None306 for e in events:307 d = e['data']308 if isinstance(d, dict):309 shapes.append({'object': d.get('object'), 'choices': [{'delta': c.get('delta'), 'finish_reason': c.get('finish_reason')} for c in d.get('choices', [])], 'usage': d.get('usage') is not None, 'obfuscation': 'obfuscation' in d})310 if d.get('usage'):311 usage = d['usage']312 else:313 shapes.append(d)314 record('l2-chat-stream', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"n_chunks={len(events)} last={events[-1]['data'] if events else None} chunk_shapes={shapes[:3]}...", cost_from_usage(C, usage))315 schema = {'type': 'object', 'properties': {'answer': {'type': 'string'}}, 'required': ['answer'], 'additionalProperties': False}316 body = {'model': C, 'messages': msgs, 'max_completion_tokens': 32, 'response_format': {'type': 'json_schema', 'json_schema': {'name': 'ok_reply', 'strict': True, 'schema': schema}}}317 st, out, _ = call('l3-chat-structured-output', 'POST', '/v1/chat/completions', body, C)318 record('l3-chat-structured-output', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"content={out['choices'][0]['message'].get('content')!r}" if st == 200 else json.dumps(out)[:300], cost_from_usage(C, out.get('usage')) if st == 200 else 0)319 body = {'model': C, 'messages': msgs, 'max_completion_tokens': 32, 'store': True, 'metadata': {'atlas': 'openai-core'}}320 st, out, _ = call('l4-chat-store-true', 'POST', '/v1/chat/completions', body, C)321 if st != 200:322 record('l4-chat-store-true', 'FAILED_VERIFICATION', st, json.dumps(out)[:300]); return323 ccid = out['id']324 record('l4-chat-store-true', 'LIVE_VERIFIED', st, f"id_prefix={ccid[:8]}", cost_from_usage(C, out.get('usage')))325 time.sleep(2)326 st, out, _ = call('l5-chat-retrieve', 'GET', f'/v1/chat/completions/{ccid}', None, C, est=0)327 record('l5-chat-retrieve', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"keys={sorted(out.keys())}" if st == 200 else json.dumps(out)[:300])328 st, out, _ = call('l6-chat-messages', 'GET', f'/v1/chat/completions/{ccid}/messages?limit=5', None, C, est=0)329 record('l6-chat-messages', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"object={out.get('object')} n={len(out.get('data', []))} first={out['data'][0] if out.get('data') else None}" if st == 200 else json.dumps(out)[:300])330 st, out, _ = call('l7-chat-list', 'GET', '/v1/chat/completions?limit=1&metadata[atlas]=openai-core', None, C, est=0)331 record('l7-chat-list', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"object={out.get('object')} n={len(out.get('data', []))} has_more={out.get('has_more')}" if st == 200 else json.dumps(out)[:300])332 st, out, _ = call('l8-chat-update', 'POST', f'/v1/chat/completions/{ccid}', {'metadata': {'atlas': 'openai-core', 'updated': 'yes'}}, C, est=0)333 record('l8-chat-update', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"metadata={out.get('metadata')}" if st == 200 else json.dumps(out)[:300])334 st, out, _ = call('l9-chat-delete', 'DELETE', f'/v1/chat/completions/{ccid}', None, C, est=0)335 record('l9-chat-delete', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, json.dumps(out)[:200])336 # legacy params on chat: max_tokens + functions (deprecated) — expect still accepted337 body = {'model': C, 'messages': msgs, 'max_tokens': 16, 'n': 1, 'seed': 7, 'logprobs': True, 'top_logprobs': 1, 'user': 'atlas-user'}338 st, out, _ = call('l10-chat-legacy-params', 'POST', '/v1/chat/completions', body, C)339 record('l10-chat-legacy-params', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"system_fingerprint={out.get('system_fingerprint')} logprobs_present={out['choices'][0].get('logprobs') is not None}" if st == 200 else json.dumps(out)[:300], cost_from_usage(C, out.get('usage')) if st == 200 else 0)340341# ---------------------------------------------------------------- (m) legacy completions342def t_m():343 body = {'model': L, 'prompt': 'Say OK.', 'max_tokens': 5}344 st, out, _ = call('m-completions-legacy', 'POST', '/v1/completions', body, L)345 record('m-completions-legacy', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"object={out.get('object')} text={out['choices'][0].get('text')!r} finish={out['choices'][0].get('finish_reason')} usage={out.get('usage')}" if st == 200 else json.dumps(out)[:300], cost_from_usage(L, out.get('usage')) if st == 200 else 0)346347# ---------------------------------------------------------------- (n) prompt caching348def t_n():349 prefix = ('You are a meticulous assistant. ' + 'The quick brown fox jumps over the lazy dog near the riverbank at dawn. ') * 70 # ~1100 tokens350 msgs = [{'role': 'system', 'content': prefix}, {'role': 'user', 'content': 'Reply with OK.'}]351 body = {'model': C, 'messages': msgs, 'max_completion_tokens': 8, 'prompt_cache_key': 'atlas-openai-core-cache'}352 st1, out1, _ = call('n1-chat-cache-first', 'POST', '/v1/chat/completions', body, C, est=0.0002)353 time.sleep(1.5)354 st2, out2, _ = call('n2-chat-cache-second', 'POST', '/v1/chat/completions', body, C, est=0.0002)355 u1 = out1.get('usage') if st1 == 200 else None356 u2 = out2.get('usage') if st2 == 200 else None357 record('n-chat-prompt-caching', 'LIVE_VERIFIED' if st1 == 200 and st2 == 200 else 'FAILED_VERIFICATION', st2, f"first usage={u1} second usage={u2}", cost_from_usage(C, u1) + cost_from_usage(C, u2))358 body = {'model': R, 'instructions': prefix, 'input': 'Reply with OK.', 'max_output_tokens': 16, 'prompt_cache_key': 'atlas-openai-core-cache-r', 'reasoning': {'effort': 'low'}, 'store': False}359 st1, out1, _ = call('n3-responses-cache-first', 'POST', '/v1/responses', body, R, est=0.0003)360 time.sleep(1.5)361 st2, out2, _ = call('n4-responses-cache-second', 'POST', '/v1/responses', body, R, est=0.0003)362 u1 = out1.get('usage') if st1 == 200 else None363 u2 = out2.get('usage') if st2 == 200 else None364 record('n-responses-prompt-caching', 'LIVE_VERIFIED' if st1 == 200 and st2 == 200 else 'FAILED_VERIFICATION', st2, f"first usage={u1} second usage={u2} prompt_cache_retention_echo={out2.get('prompt_cache_retention') if st2 == 200 else None} prompt_cache_key_echo={out2.get('prompt_cache_key') if st2 == 200 else None}", cost_from_usage(R, u1) + cost_from_usage(R, u2))365366# ---------------------------------------------------------------- (o) image input367def t_o():368 body = {'model': C, 'max_output_tokens': 32, 'input': [{'role': 'user', 'content': [{'type': 'input_text', 'text': 'Reply with OK.'}, {'type': 'input_image', 'image_url': PNG_1x1, 'detail': 'low'}]}]}369 st, out, _ = call('o1-responses-image-input', 'POST', '/v1/responses', body, C)370 record('o1-responses-image-input', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"status={out.get('status')} usage={out.get('usage')}" if st == 200 else json.dumps(out)[:300], cost_from_usage(C, out.get('usage')) if st == 200 else 0)371 body = {'model': C, 'max_completion_tokens': 32, 'messages': [{'role': 'user', 'content': [{'type': 'text', 'text': 'Reply with OK.'}, {'type': 'image_url', 'image_url': {'url': PNG_1x1, 'detail': 'low'}}]}]}372 st, out, _ = call('o2-chat-image-input', 'POST', '/v1/chat/completions', body, C)373 record('o2-chat-image-input', 'LIVE_VERIFIED' if st == 200 else 'FAILED_VERIFICATION', st, f"usage={out.get('usage')}" if st == 200 else json.dumps(out)[:300], cost_from_usage(C, out.get('usage')) if st == 200 else 0)374375376if __name__ == '__main__':377 only = set(sys.argv[1:])378 tests = [('a', t_a), ('a2', t_a2), ('b', t_b), ('c', t_c), ('d', t_d), ('e', t_e), ('f', t_f), ('g', t_g), ('h', t_h), ('i', t_i),379 ('j', t_j), ('k', t_k), ('l', t_l), ('m', t_m), ('n', t_n), ('o', t_o), ('gdel', t_g_delete)]380 for key, fn in tests:381 if only and key not in only:382 continue383 run(key, fn)384 print('\nTOTAL est cost: $%.5f' % total_cost)385 live.save_sanitized({'total_est_cost_usd': round(total_cost, 6), 'sse_types_b': state.get('b_types'), 'results': summary}, OUT / '_summary.json')386