Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Live probes for the OpenAI tools domain (function calling, built-in tools, containers, skills).3Raw sanitized responses -> tmp-live/tools/<name>.json ; every call logged to reports/live-requests.jsonl.4Usage: .venv/bin/python tmp-live/tools_probe.py [probe ...]5"""6from __future__ import annotations7import base64, io, json, sys, time, zipfile8from pathlib import Path9ROOT = Path(__file__).resolve().parent.parent10sys.path.insert(0, str(ROOT))11from scripts.live import openai_request, save_sanitized, mask # noqa: E4021213OUT = ROOT / "tmp-live" / "tools"14NANO = "gpt-5.4-nano"15MINI = "gpt-5.4-mini"16# rough per-token prices (USD / 1M) used only for est_cost_usd logging17PRICE = {NANO: (0.20, 1.25), MINI: (0.75, 4.50), "gpt-4.1-nano": (0.10, 0.40), "computer-use-preview": (3.0, 12.0)}181920def est(model: str, body) -> float:21 u = (body or {}).get("usage") if isinstance(body, dict) else None22 if not u:23 return 0.024 i, o = PRICE.get(model, (1.0, 5.0))25 return (u.get("input_tokens", u.get("prompt_tokens", 0)) * i + u.get("output_tokens", u.get("completion_tokens", 0)) * o) / 1e6262728def save(name: str, obj) -> None:29 save_sanitized(obj, OUT / f"{name}.json")303132def kinds(body) -> list[str]:33 if not isinstance(body, dict):34 return []35 return [f"{it.get('type')}:{it.get('status', '')}" for it in body.get("output", [])]363738def resp(name: str, payload: dict, extra_cost: float = 0.0, model: str | None = None, note: str = ""):39 model = model or payload.get("model", NANO)40 st, body, hdrs = openai_request("POST", "/v1/responses", payload, est_cost_usd=extra_cost, note=note or name)41 save(name, {"status": st, "request": payload, "response": body})42 c = est(model, body) + extra_cost43 print(f"[{name}] HTTP {st} model={model} cost~${c:.4f} output={kinds(body)}"44 + (f" ERROR={json.dumps(body.get('error'))[:300]}" if isinstance(body, dict) and body.get("error") else ""))45 return st, body464748def stream(name: str, payload: dict):49 payload = {**payload, "stream": True}50 st, lines, hdrs = openai_request("POST", "/v1/responses", payload, stream=True, note=name + " (stream)")51 events, samples = [], {}52 if st != 200:53 body = b"".join(l.encode() for l in lines) if not isinstance(lines, (dict, bytes)) else lines54 print(f"[{name}] HTTP {st} {str(body)[:300]}")55 return st, []56 for line in lines:57 if line.startswith("data: "):58 try:59 ev = json.loads(line[6:])60 except Exception:61 continue62 t = ev.get("type")63 events.append(t)64 samples.setdefault(t, ev)65 save(name, {"status": st, "request": payload, "event_sequence": events, "samples": samples})66 print(f"[{name}] HTTP {st} events={len(events)} distinct={sorted(set(events))}")67 return st, events686970WEATHER = {"type": "function", "name": "get_weather", "description": "Get current weather for a city.",71 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"],72 "additionalProperties": False}, "strict": True}737475def p_function():76 st, body = resp("function_call_forced", {"model": NANO, "input": "What is the weather in Paris?",77 "tools": [WEATHER], "tool_choice": {"type": "function", "name": "get_weather"},78 "max_output_tokens": 64, "reasoning": {"effort": "low"}})79 call = next((it for it in body.get("output", []) if it.get("type") == "function_call"), None)80 if call:81 print(" function_call item keys:", sorted(call.keys()), "args=", call.get("arguments"))82 resp("function_call_output_roundtrip", {"model": NANO, "previous_response_id": body["id"],83 "input": [{"type": "function_call_output", "call_id": call["call_id"],84 "output": json.dumps({"temp_c": 18, "sky": "clear"})}],85 "tools": [WEATHER], "max_output_tokens": 32, "reasoning": {"effort": "low"}})86 stream("function_call_stream", {"model": NANO, "input": "Weather in Rome?", "tools": [WEATHER],87 "tool_choice": "required", "max_output_tokens": 64, "reasoning": {"effort": "low"}})88 # strict schema violation: additionalProperties missing -> expect 40089 bad = {**WEATHER, "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}90 resp("function_strict_invalid_schema", {"model": NANO, "input": "hi", "tools": [bad], "max_output_tokens": 16})91 # parallel_tool_calls false + tool_choice none + allowed_tools92 resp("function_allowed_tools", {"model": NANO, "input": "Weather in Oslo?", "tools": [WEATHER, {**WEATHER, "name": "get_time"}],93 "tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]},94 "parallel_tool_calls": False, "max_output_tokens": 64, "reasoning": {"effort": "low"}})95 # namespace96 resp("function_namespace", {"model": NANO, "input": "Weather in Lima?", "tool_choice": "required",97 "tools": [{"type": "namespace", "name": "weather", "description": "Weather tools", "tools": [WEATHER]}],98 "max_output_tokens": 64, "reasoning": {"effort": "low"}})99 # chat completions function call (gpt-4.1-nano)100 st, body, _ = openai_request("POST", "/v1/chat/completions", {101 "model": "gpt-4.1-nano", "messages": [{"role": "user", "content": "Weather in Paris?"}],102 "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": WEATHER["parameters"], "strict": True}}],103 "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, "max_tokens": 32}, note="chat completions function call")104 save("chat_function_call", {"status": st, "response": body})105 print(f"[chat_function_call] HTTP {st}", json.dumps(body.get("choices", [{}])[0].get("message", {}).get("tool_calls"))[:300] if isinstance(body, dict) else body)106107108def p_custom():109 tool = {"type": "custom", "name": "answer_yes_no", "description": "Reply with exactly yes or no.",110 "format": {"type": "grammar", "syntax": "regex", "definition": r"^(yes|no)$"}}111 for m in (NANO, MINI):112 st, body = resp(f"custom_grammar_{m}", {"model": m, "input": "Is 2+2 equal to 4? Use the tool.", "tools": [tool],113 "tool_choice": {"type": "custom", "name": "answer_yes_no"},114 "max_output_tokens": 32, "reasoning": {"effort": "low"}})115 if st == 200:116 break117 resp("custom_text_format", {"model": NANO, "input": "Say OK via the tool.", "tool_choice": "required",118 "tools": [{"type": "custom", "name": "echo", "description": "Echo free text."}],119 "max_output_tokens": 32, "reasoning": {"effort": "low"}})120121122def p_web_search():123 resp("web_search", {"model": NANO, "input": "What is today's date in Toronto? Reply in 5 words.",124 "tools": [{"type": "web_search", "search_context_size": "low",125 "user_location": {"type": "approximate", "country": "CA", "city": "Toronto"}}],126 "include": ["web_search_call.action.sources"], "max_output_tokens": 64, "reasoning": {"effort": "low"}},127 extra_cost=0.01)128129130def p_code_interpreter():131 st, body = resp("code_interpreter", {"model": NANO, "input": "Run print(2+2) in python and reply with just the number.",132 "tools": [{"type": "code_interpreter", "container": {"type": "auto"}}],133 "include": ["code_interpreter_call.outputs"], "max_output_tokens": 64,134 "reasoning": {"effort": "low"}}, extra_cost=0.03)135 ci = next((it for it in body.get("output", []) if it.get("type") == "code_interpreter_call"), None) if st == 200 else None136 if ci:137 cid = ci["container_id"]138 print(" container_id", cid, "code=", repr(ci.get("code"))[:80], "outputs=", json.dumps(ci.get("outputs"))[:200])139 st, c, _ = openai_request("GET", f"/v1/containers/{cid}", note="retrieve container"); save("container_retrieve", c); print(" GET container", st, json.dumps(c)[:300])140 st, f, _ = openai_request("GET", f"/v1/containers/{cid}/files", note="list container files"); save("container_files_list", f); print(" GET files", st, json.dumps(f)[:300])141 st, l, _ = openai_request("GET", "/v1/containers?limit=2", note="list containers"); save("containers_list", l); print(" GET containers", st, [x.get("id") for x in l.get("data", [])] if isinstance(l, dict) else l)142 st, d, _ = openai_request("DELETE", f"/v1/containers/{cid}", note="delete container"); save("container_delete", d); print(" DELETE container", st, json.dumps(d)[:200])143 # explicit container create/delete (no model call) to record create shape144 st, c, _ = openai_request("POST", "/v1/containers", {"name": "atlas-tools-agent", "memory_limit": "1g",145 "expires_after": {"anchor": "last_active_at", "minutes": 5}}, note="create container")146 save("container_create", c); print(" POST containers", st, json.dumps(c)[:300])147 if st == 200:148 cid = c["id"]149 st, cf, _ = openai_request("POST", f"/v1/containers/{cid}/files", data=(150 b"--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"hello.txt\"\r\nContent-Type: text/plain\r\n\r\nhello atlas\r\n--b--\r\n"),151 content_type="multipart/form-data; boundary=b", note="create container file")152 save("container_file_create", cf); print(" POST container file", st, json.dumps(cf)[:300])153 if st == 200:154 fid = cf["id"]155 st, r, _ = openai_request("GET", f"/v1/containers/{cid}/files/{fid}", note="retrieve container file"); save("container_file_retrieve", r); print(" GET file", st)156 st, r, _ = openai_request("GET", f"/v1/containers/{cid}/files/{fid}/content", note="container file content"); print(" GET content", st, r[:40] if isinstance(r, bytes) else r)157 st, r, _ = openai_request("DELETE", f"/v1/containers/{cid}/files/{fid}", note="delete container file"); save("container_file_delete", r); print(" DELETE file", st, json.dumps(r)[:200])158 st, d, _ = openai_request("DELETE", f"/v1/containers/{cid}", note="delete container"); print(" DELETE container", st)159160161def p_image_invalid():162 resp("image_generation_invalid_size", {"model": NANO, "input": "Draw a red dot.",163 "tools": [{"type": "image_generation", "size": "1x1"}], "max_output_tokens": 16})164165166def p_mcp():167 resp("mcp_deepwiki", {"model": NANO, "input": "Use the deepwiki tool read_wiki_structure for repo openai/openai-python and reply with the first topic name only.",168 "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp",169 "require_approval": "never", "allowed_tools": ["read_wiki_structure"]}],170 "max_output_tokens": 64, "reasoning": {"effort": "low"}})171 # approval flow: require_approval always -> expect mcp_approval_request172 st, body = resp("mcp_approval_request", {"model": NANO, "input": "Call read_wiki_structure for openai/openai-python.",173 "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp",174 "require_approval": "always", "allowed_tools": ["read_wiki_structure"]}],175 "tool_choice": {"type": "mcp", "server_label": "deepwiki", "name": "read_wiki_structure"},176 "max_output_tokens": 64, "reasoning": {"effort": "low"}})177 ar = next((it for it in body.get("output", []) if it.get("type") == "mcp_approval_request"), None) if st == 200 else None178 if ar:179 resp("mcp_approval_denied", {"model": NANO, "previous_response_id": body["id"],180 "input": [{"type": "mcp_approval_response", "approval_request_id": ar["id"], "approve": False, "reason": "atlas test"}],181 "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp",182 "require_approval": "always", "allowed_tools": ["read_wiki_structure"]}],183 "max_output_tokens": 32, "reasoning": {"effort": "low"}})184185186PNG1 = base64.b64encode(bytes.fromhex(187 "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d4944415478da63f8ffff3f0300"188 "0500020a2c2d3e850000000049454e44ae426082")).decode()189190191def p_computer():192 resp("computer_use_preview", {"model": "computer-use-preview",193 "input": [{"role": "user", "content": [{"type": "input_text", "text": "Do nothing. Reply with OK."},194 {"type": "input_image", "image_url": f"data:image/png;base64,{PNG1}"}]}],195 "tools": [{"type": "computer_use_preview", "display_width": 1, "display_height": 1, "environment": "browser"}],196 "truncation": "auto", "max_output_tokens": 32}, model="computer-use-preview")197198199def p_skills():200 st, body, _ = openai_request("GET", "/v1/skills?limit=5", note="list skills"); save("skills_list", body); print("[skills_list]", st, json.dumps(body)[:300])201 buf = io.BytesIO()202 with zipfile.ZipFile(buf, "w") as z:203 z.writestr("atlas-hello/SKILL.md", "---\nname: atlas-hello\ndescription: Atlas test skill. Prints OK when asked.\n---\n\n# Atlas hello\n\nWhen asked, run `echo OK`.\n")204 zip_bytes = buf.getvalue()205 mp = (b"--b\r\nContent-Disposition: form-data; name=\"files\"; filename=\"atlas-hello.zip\"\r\nContent-Type: application/zip\r\n\r\n"206 + zip_bytes + b"\r\n--b--\r\n")207 st, sk, _ = openai_request("POST", "/v1/skills", data=mp, content_type="multipart/form-data; boundary=b", note="create skill (zip)")208 save("skill_create", sk); print("[skill_create]", st, json.dumps(sk)[:400])209 if st != 200:210 return211 sid = sk["id"]212 st, r, _ = openai_request("GET", f"/v1/skills/{sid}", note="get skill"); save("skill_get", r); print("[skill_get]", st)213 st, r, _ = openai_request("GET", f"/v1/skills/{sid}/versions", note="list skill versions"); save("skill_versions", r); print("[skill_versions]", st, json.dumps(r)[:300])214 st, r, _ = openai_request("GET", f"/v1/skills/{sid}/versions/1", note="get skill version"); save("skill_version_get", r); print("[skill_version_get]", st)215 st, r, _ = openai_request("GET", f"/v1/skills/{sid}/content", note="skill content"); print("[skill_content]", st, (r[:4] if isinstance(r, bytes) else r))216 st, r, _ = openai_request("POST", f"/v1/skills/{sid}/versions", data=mp, content_type="multipart/form-data; boundary=b", note="create skill version"); save("skill_version_create", r); print("[skill_version_create]", st, json.dumps(r)[:300])217 st, r, _ = openai_request("POST", f"/v1/skills/{sid}", {"default_version": "2"}, note="set default version"); save("skill_set_default", r); print("[skill_set_default]", st, json.dumps(r)[:200])218 st, r, _ = openai_request("DELETE", f"/v1/skills/{sid}/versions/1", note="delete skill version"); save("skill_version_delete", r); print("[skill_version_delete]", st, json.dumps(r)[:200])219 # use the skill in hosted shell (nano supports hosted_shell + skills per model page)220 resp("shell_hosted_skill", {"model": NANO, "input": "Use the atlas-hello skill: run echo OK and reply with the output only.",221 "tools": [{"type": "shell", "environment": {"type": "container_auto", "skills": [{"type": "skill_reference", "skill_id": sid}]}}],222 "max_output_tokens": 128, "reasoning": {"effort": "low"}}, extra_cost=0.03)223 st, r, _ = openai_request("DELETE", f"/v1/skills/{sid}", note="delete skill"); save("skill_delete", r); print("[skill_delete]", st, json.dumps(r)[:200])224225226def p_shell_patch():227 resp("shell_local", {"model": NANO, "input": "Propose the shell command `echo OK`. Do not explain.",228 "tools": [{"type": "shell", "environment": {"type": "local"}}], "tool_choice": {"type": "shell"},229 "max_output_tokens": 64, "reasoning": {"effort": "low"}})230 resp("apply_patch", {"model": NANO, "input": "Create a new file hello.txt containing the single line OK using apply_patch.",231 "tools": [{"type": "apply_patch"}], "tool_choice": {"type": "apply_patch"},232 "max_output_tokens": 128, "reasoning": {"effort": "low"}})233 resp("local_shell_nano", {"model": NANO, "input": "Propose `echo OK`.", "tools": [{"type": "local_shell"}], "max_output_tokens": 32})234235236def p_tool_search():237 deferred = {**WEATHER, "defer_loading": True}238 resp("tool_search_mini", {"model": MINI, "input": "Weather in Paris? Find and use a tool.",239 "tools": [{"type": "tool_search"}, {"type": "namespace", "name": "weather", "description": "Weather lookup tools", "tools": [deferred]}],240 "max_output_tokens": 96, "reasoning": {"effort": "low"}})241242243def p_file_search():244 st, vs, _ = openai_request("POST", "/v1/vector_stores", {"name": "atlas-tools-agent"}, note="create vector store (tools agent)")245 save("vs_create", vs); print("[vs_create]", st, vs.get("id") if isinstance(vs, dict) else vs)246 if st != 200:247 return248 vsid = vs["id"]249 content = b"Atlas fact sheet. The secret codeword for the API Atlas tools agent is PELICAN-42. Nothing else matters here. Filler text to reach two hundred bytes of content for the test file ok."250 mp = (b"--b\r\nContent-Disposition: form-data; name=\"purpose\"\r\n\r\nassistants\r\n--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"atlas.txt\"\r\nContent-Type: text/plain\r\n\r\n" + content + b"\r\n--b--\r\n")251 st, f, _ = openai_request("POST", "/v1/files", data=mp, content_type="multipart/form-data; boundary=b", note="upload file for file_search")252 fid = f.get("id"); print("[file_upload]", st, fid)253 st, vf, _ = openai_request("POST", f"/v1/vector_stores/{vsid}/files", {"file_id": fid}, note="attach file to vector store")254 for _ in range(20):255 st, vf, _ = openai_request("GET", f"/v1/vector_stores/{vsid}/files/{fid}", note="poll vs file status")256 if vf.get("status") in ("completed", "failed"):257 break258 time.sleep(1.5)259 print("[vs_file]", vf.get("status"))260 resp("file_search", {"model": NANO, "input": "What is the secret codeword? Reply with the codeword only.",261 "tools": [{"type": "file_search", "vector_store_ids": [vsid], "max_num_results": 2}],262 "include": ["file_search_call.results"], "max_output_tokens": 32, "reasoning": {"effort": "low"}}, extra_cost=0.0025)263 st, d, _ = openai_request("DELETE", f"/v1/vector_stores/{vsid}", note="delete vector store"); print("[vs_delete]", st)264 st, d, _ = openai_request("DELETE", f"/v1/files/{fid}", note="delete file"); print("[file_delete]", st)265266267def p_retry_streams():268 # web search with enough output budget, streamed to capture events + final response269 stream("web_search_stream", {"model": NANO, "input": "What is today's date in Toronto? Reply in 5 words.",270 "tools": [{"type": "web_search", "search_context_size": "low",271 "user_location": {"type": "approximate", "country": "CA", "city": "Toronto"}}],272 "include": ["web_search_call.action.sources"], "max_output_tokens": 600, "reasoning": {"effort": "low"}})273 from scripts.live import log_request274 log_request("openai", "POST", "/v1/responses", 200, 0.01, "web search per-call fee ($10/1k)")275 # file search retry276 st, vs, _ = openai_request("POST", "/v1/vector_stores", {"name": "atlas-tools-agent"}, note="create vector store (tools agent, retry)")277 vsid = vs["id"]278 content = b"Atlas fact sheet. The secret codeword for the API Atlas tools agent is PELICAN-42. Nothing else matters here. Filler text to reach two hundred bytes of content for the test file ok."279 mp = (b"--b\r\nContent-Disposition: form-data; name=\"purpose\"\r\n\r\nassistants\r\n--b\r\nContent-Disposition: form-data; name=\"file\"; filename=\"atlas.txt\"\r\nContent-Type: text/plain\r\n\r\n" + content + b"\r\n--b--\r\n")280 st, f, _ = openai_request("POST", "/v1/files", data=mp, content_type="multipart/form-data; boundary=b", note="upload file for file_search")281 fid = f["id"]282 openai_request("POST", f"/v1/vector_stores/{vsid}/files", {"file_id": fid}, note="attach file")283 for _ in range(20):284 st, vf, _ = openai_request("GET", f"/v1/vector_stores/{vsid}/files/{fid}", note="poll")285 if vf.get("status") in ("completed", "failed"):286 break287 time.sleep(1.5)288 stream("file_search_stream", {"model": NANO, "input": "What is the secret codeword? Reply with the codeword only.",289 "tools": [{"type": "file_search", "vector_store_ids": [vsid], "max_num_results": 2}],290 "include": ["file_search_call.results"], "max_output_tokens": 300, "reasoning": {"effort": "low"}})291 log_request("openai", "POST", "/v1/responses", 200, 0.0025, "file search per-call fee ($2.50/1k)")292 openai_request("DELETE", f"/v1/vector_stores/{vsid}", note="delete vector store")293 openai_request("DELETE", f"/v1/files/{fid}", note="delete file")294 # cheap streams for event capture295 stream("mcp_stream", {"model": NANO, "input": "Use read_wiki_structure for openai/openai-python; reply with the first topic only.",296 "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp",297 "require_approval": "never", "allowed_tools": ["read_wiki_structure"]}],298 "max_output_tokens": 64, "reasoning": {"effort": "low"}})299 stream("custom_stream", {"model": NANO, "input": "Is 2+2 equal to 4? Use the tool.", "tool_choice": {"type": "custom", "name": "answer_yes_no"},300 "tools": [{"type": "custom", "name": "answer_yes_no", "format": {"type": "grammar", "syntax": "regex", "definition": r"^(yes|no)$"}}],301 "max_output_tokens": 32, "reasoning": {"effort": "low"}})302 stream("shell_local_stream", {"model": NANO, "input": "Propose the shell command `echo OK`.", "tool_choice": {"type": "shell"},303 "tools": [{"type": "shell", "environment": {"type": "local"}}], "max_output_tokens": 64, "reasoning": {"effort": "low"}})304 stream("apply_patch_stream", {"model": NANO, "input": "Create hello.txt containing OK using apply_patch.", "tool_choice": {"type": "apply_patch"},305 "tools": [{"type": "apply_patch"}], "max_output_tokens": 128, "reasoning": {"effort": "low"}})306307308PROBES = {"retry": p_retry_streams, "function": p_function, "custom": p_custom, "web_search": p_web_search, "code_interpreter": p_code_interpreter,309 "image": p_image_invalid, "mcp": p_mcp, "computer": p_computer, "skills": p_skills, "shell_patch": p_shell_patch,310 "tool_search": p_tool_search, "file_search": p_file_search}311312if __name__ == "__main__":313 OUT.mkdir(parents=True, exist_ok=True)314 for name in (sys.argv[1:] or PROBES):315 print(f"===== {name}")316 try:317 PROBES[name]()318 except Exception as e: # noqa: BLE001319 print(f"[{name}] EXCEPTION {mask(repr(e))[:300]}")320