Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Live probes for the Anthropic tool-use ecosystem (API Atlas, 2026-09-18).3Every call goes through scripts.live.anthropic_request (auto-logged + key masking).4Raw sanitized responses are written to tmp-live/tools/<step>.json.5Budget target: <= $0.80 total.6"""7from __future__ import annotations8import json, sys, time, base649from pathlib import Path10ROOT = Path(__file__).resolve().parents[2]11sys.path.insert(0, str(ROOT))12from scripts import live # noqa: E4021314OUT = ROOT / "tmp-live" / "tools"15OUT.mkdir(parents=True, exist_ok=True)16HAIKU = "claude-haiku-4-5-20251001"17SONNET46 = "claude-sonnet-4-6"18SONNET5 = "claude-sonnet-5"19PRICE = {HAIKU: (1.0, 5.0), SONNET46: (3.0, 15.0), SONNET5: (2.0, 10.0)} # $/MTok in, out20TOTAL = 0.021SUMMARY: list[dict] = []22ONLY = set(sys.argv[1:])232425def cost(model, usage, extra=0.0):26 if not isinstance(usage, dict):27 return extra28 pi, po = PRICE.get(model, (3.0, 15.0))29 c = usage.get("input_tokens", 0) * pi / 1e6 + usage.get("output_tokens", 0) * po / 1e630 c += usage.get("cache_creation_input_tokens", 0) * pi * 1.25 / 1e6 + usage.get("cache_read_input_tokens", 0) * pi * 0.1 / 1e631 stu = usage.get("server_tool_use") or {}32 c += stu.get("web_search_requests", 0) * 0.0133 return c + extra343536def call(step, body, beta=None, note="", stream=False, path="/v1/messages"):37 """POST and persist. Returns (status, body_json)."""38 global TOTAL39 if ONLY and step not in ONLY:40 return None, None41 t0 = time.time()42 st, out, hdrs = live.anthropic_request("POST", path, body, beta=beta, note=f"tools:{step} {note}", stream=stream)43 if stream:44 lines = list(out)45 out = {"_sse_lines": lines}46 # parse events for usage47 usage = {}48 for ln in lines:49 if ln.startswith("data: "):50 try:51 ev = json.loads(ln[6:])52 except Exception:53 continue54 if ev.get("type") == "message_start":55 usage.update(ev["message"].get("usage", {}))56 if ev.get("type") == "message_delta":57 usage["output_tokens"] = ev.get("usage", {}).get("output_tokens", usage.get("output_tokens", 0))58 out["_usage"] = usage59 else:60 usage = out.get("usage", {}) if isinstance(out, dict) else {}61 c = cost(body.get("model", ""), usage) if st == 200 else 0.062 TOTAL += c63 rec = {"step": step, "status": st, "model": body.get("model"), "beta": beta, "est_cost_usd": round(c, 5),64 "secs": round(time.time() - t0, 1), "request_id": hdrs.get("request-id"),65 "stop_reason": out.get("stop_reason") if isinstance(out, dict) else None}66 if st != 200 and isinstance(out, dict):67 rec["error"] = out.get("error")68 SUMMARY.append(rec)69 live.save_sanitized({"request": body, "beta": beta, "status": st, "response": out,70 "headers": live.interesting_headers(hdrs)}, OUT / f"{step}.json")71 print(json.dumps(rec), flush=True)72 return st, out737475def blocks(out, t):76 return [b for b in (out or {}).get("content", []) if b.get("type") == t]777879WEATHER = {"name": "get_weather", "description": "Get the current weather for a city. Returns a short text description.",80 "input_schema": {"type": "object", "properties": {"location": {"type": "string", "description": "City name, e.g. Paris"}},81 "required": ["location"]}}82TIME = {"name": "get_time", "description": "Get the current local time in a city.",83 "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}8485# ---------------- (a) custom tools ----------------86st, r = call("a1_forced_tool", {"model": HAIKU, "max_tokens": 200, "tools": [WEATHER],87 "tool_choice": {"type": "tool", "name": "get_weather"},88 "messages": [{"role": "user", "content": "Weather in Paris?"}]})89if st == 200 and blocks(r, "tool_use"):90 tu = blocks(r, "tool_use")[0]91 call("a2_tool_result_final", {"model": HAIKU, "max_tokens": 60, "tools": [WEATHER],92 "messages": [{"role": "user", "content": "Weather in Paris?"},93 {"role": "assistant", "content": r["content"]},94 {"role": "user", "content": [{"type": "tool_result", "tool_use_id": tu["id"],95 "content": "18 C, light rain"}]}]})96call("a3_parallel_two_tools", {"model": HAIKU, "max_tokens": 300, "tools": [WEATHER, TIME],97 "messages": [{"role": "user", "content": "Give me the weather AND the local time in Tokyo. Call both tools at once."}]})98call("a4_disable_parallel", {"model": HAIKU, "max_tokens": 300, "tools": [WEATHER, TIME],99 "tool_choice": {"type": "auto", "disable_parallel_tool_use": True},100 "messages": [{"role": "user", "content": "Give me the weather AND the local time in Tokyo. Call both tools at once."}]})101call("a5_tool_choice_any", {"model": HAIKU, "max_tokens": 150, "tools": [WEATHER, TIME], "tool_choice": {"type": "any"},102 "messages": [{"role": "user", "content": "Say hello."}]})103call("a6_tool_choice_none", {"model": HAIKU, "max_tokens": 40, "tools": [WEATHER], "tool_choice": {"type": "none"},104 "messages": [{"role": "user", "content": "Weather in Paris? Reply in 5 words."}]})105STRICT = dict(WEATHER, strict=True, input_schema={"type": "object", "properties": {"location": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}},106 "required": ["location", "unit"], "additionalProperties": False})107call("a7_strict_any", {"model": HAIKU, "max_tokens": 150, "tools": [STRICT], "tool_choice": {"type": "any"},108 "messages": [{"role": "user", "content": "Weather in Rome in celsius"}]})109call("a8_is_error_result", {"model": HAIKU, "max_tokens": 60, "tools": [WEATHER],110 "messages": [{"role": "user", "content": "Weather in Paris?"},111 {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_atlas_fake_01", "name": "get_weather", "input": {"location": "Paris"}}]},112 {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_atlas_fake_01", "is_error": True, "content": "Weather service unavailable (HTTP 503)"}]}]})113call("a9_text_before_tool_result_400", {"model": HAIKU, "max_tokens": 20, "tools": [WEATHER],114 "messages": [{"role": "user", "content": "Weather in Paris?"},115 {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_atlas_fake_02", "name": "get_weather", "input": {"location": "Paris"}}]},116 {"role": "user", "content": [{"type": "text", "text": "Here:"}, {"type": "tool_result", "tool_use_id": "toolu_atlas_fake_02", "content": "18 C"}]}]})117call("a10_input_examples", {"model": HAIKU, "max_tokens": 120, "tools": [dict(WEATHER, input_examples=[{"location": "Paris"}, {"location": "Tokyo"}])],118 "tool_choice": {"type": "any"}, "messages": [{"role": "user", "content": "Weather in Oslo"}]})119120# ---------------- (b) streaming ----------------121call("b1_stream_input_json_delta", {"model": HAIKU, "max_tokens": 150, "stream": True, "tools": [WEATHER], "tool_choice": {"type": "any"},122 "messages": [{"role": "user", "content": "Weather in San Francisco, CA?"}]}, stream=True)123call("b2_stream_eager_input_streaming", {"model": HAIKU, "max_tokens": 150, "stream": True, "tools": [dict(WEATHER, eager_input_streaming=True)],124 "tool_choice": {"type": "any"}, "messages": [{"role": "user", "content": "Weather in San Francisco, CA?"}]}, stream=True)125call("b3_stream_legacy_fgts_header", {"model": HAIKU, "max_tokens": 150, "stream": True, "tools": [WEATHER], "tool_choice": {"type": "any"},126 "messages": [{"role": "user", "content": "Weather in San Francisco, CA?"}]},127 beta="fine-grained-tool-streaming-2025-05-14", stream=True)128129# ---------------- (c) web search ----------------130call("c1_web_search_20250305", {"model": HAIKU, "max_tokens": 300,131 "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 1,132 "user_location": {"type": "approximate", "city": "Toronto", "region": "Ontario", "country": "CA", "timezone": "America/Toronto"}}],133 "messages": [{"role": "user", "content": "What is today's date in Toronto? Answer in 5 words."}]})134call("c2_web_search_20260209_haiku_no_allowed_callers", {"model": HAIKU, "max_tokens": 50,135 "tools": [{"type": "web_search_20260209", "name": "web_search", "max_uses": 1}],136 "messages": [{"role": "user", "content": "Reply with OK."}]})137call("c3_web_search_both_domain_lists_400", {"model": HAIKU, "max_tokens": 20,138 "tools": [{"type": "web_search_20250305", "name": "web_search", "allowed_domains": ["example.com"], "blocked_domains": ["foo.com"]}],139 "messages": [{"role": "user", "content": "Reply with OK."}]})140141# ---------------- (d) web fetch ----------------142call("d1_web_fetch_20250910", {"model": HAIKU, "max_tokens": 200,143 "tools": [{"type": "web_fetch_20250910", "name": "web_fetch", "max_uses": 1, "citations": {"enabled": True}, "max_content_tokens": 2000}],144 "messages": [{"role": "user", "content": "Fetch https://example.com and tell me the page title in 3 words."}]})145146# ---------------- (e) code execution ----------------147st, r = call("e1_code_execution_20250825", {"model": HAIKU, "max_tokens": 400,148 "tools": [{"type": "code_execution_20250825", "name": "code_execution"}],149 "messages": [{"role": "user", "content": "Use the code execution tool to run print(2+2) and reply with only the number."}]})150if st == 200 and isinstance(r, dict) and r.get("container"):151 cid = r["container"]["id"]152 call("e2_code_execution_container_reuse", {"model": HAIKU, "max_tokens": 300, "container": cid,153 "tools": [{"type": "code_execution_20250825", "name": "code_execution"}],154 "messages": [{"role": "user", "content": "Run the shell command `echo hi` with the code execution tool and reply with only its output."}]})155call("e3_code_execution_20260521_haiku", {"model": HAIKU, "max_tokens": 300,156 "tools": [{"type": "code_execution_20260521", "name": "code_execution"}],157 "messages": [{"role": "user", "content": "Use the code execution tool to run print(3*3) and reply with only the number."}]})158159# ---------------- (f) tool search ----------------160DUMMY = []161for n, d in [("get_weather", "Get the current weather for a city"), ("get_stock_price", "Get the latest stock price for a ticker symbol"),162 ("send_email", "Send an email to a recipient"), ("create_calendar_event", "Create a calendar event"),163 ("translate_text", "Translate text into a target language"), ("lookup_zip_code", "Look up the city for a US zip code")]:164 DUMMY.append({"name": n, "description": d, "defer_loading": True,165 "input_schema": {"type": "object", "properties": {"q": {"type": "string", "description": "Main argument"}}, "required": ["q"]}})166call("f1_tool_search_regex", {"model": HAIKU, "max_tokens": 300,167 "tools": [{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}] + DUMMY,168 "messages": [{"role": "user", "content": "What's the weather in Paris? Find and call the right tool."}]})169call("f2_tool_search_bm25", {"model": HAIKU, "max_tokens": 300,170 "tools": [{"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}] + DUMMY,171 "messages": [{"role": "user", "content": "What's the stock price of AAPL? Find and call the right tool."}]})172call("f3_tool_search_undated_alias", {"model": HAIKU, "max_tokens": 60,173 "tools": [{"type": "tool_search_tool_regex", "name": "tool_search_tool_regex"}] + DUMMY[:2],174 "messages": [{"role": "user", "content": "Reply with OK."}]})175call("f4_all_deferred_400", {"model": HAIKU, "max_tokens": 20, "tools": DUMMY[:2],176 "messages": [{"role": "user", "content": "Reply with OK."}]})177178# ---------------- (g) programmatic tool calling ----------------179QDB = {"name": "query_database", "description": "Execute a SQL query. Returns a JSON array of row objects.",180 "input_schema": {"type": "object", "properties": {"sql": {"type": "string"}}, "required": ["sql"]},181 "allowed_callers": ["code_execution_20260120"]}182PTC_TOOLS = [{"type": "code_execution_20260120", "name": "code_execution"}, QDB]183PTC_MSG = "Use code to call query_database with sql 'SELECT 1 AS one' and tell me the value of one. One sentence."184st, r = call("g1_ptc_sonnet46", {"model": SONNET46, "max_tokens": 500, "tools": PTC_TOOLS, "messages": [{"role": "user", "content": PTC_MSG}]})185if st == 200 and isinstance(r, dict):186 pending = [b for b in blocks(r, "tool_use") if (b.get("caller") or {}).get("type", "").startswith("code_execution")]187 if pending and r.get("container"):188 results = [{"type": "tool_result", "tool_use_id": b["id"], "content": json.dumps([{"one": 1}])} for b in pending]189 call("g2_ptc_result", {"model": SONNET46, "max_tokens": 400, "container": r["container"]["id"], "tools": PTC_TOOLS,190 "messages": [{"role": "user", "content": PTC_MSG}, {"role": "assistant", "content": r["content"]},191 {"role": "user", "content": results}]})192call("g3_ptc_haiku_unsupported", {"model": HAIKU, "max_tokens": 200, "tools": PTC_TOOLS, "messages": [{"role": "user", "content": PTC_MSG}]})193call("g4_ptc_tool_choice_400", {"model": SONNET46, "max_tokens": 20, "tools": PTC_TOOLS, "tool_choice": {"type": "tool", "name": "query_database"},194 "messages": [{"role": "user", "content": PTC_MSG}]})195196# ---------------- (h) computer use / text editor / bash / memory ----------------197PNG1x1 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="198CU = {"type": "computer_20251124", "name": "computer", "display_width_px": 1024, "display_height_px": 768, "display_number": 1}199st, r = call("h1_computer_20251124_sonnet46", {"model": SONNET46, "max_tokens": 100, "tools": [CU],200 "messages": [{"role": "user", "content": "Take a screenshot."}]}, beta="computer-use-2025-11-24")201if st == 200 and blocks(r, "tool_use"):202 tu = blocks(r, "tool_use")[0]203 call("h2_computer_screenshot_result", {"model": SONNET46, "max_tokens": 50, "tools": [CU],204 "messages": [{"role": "user", "content": "Take a screenshot."}, {"role": "assistant", "content": r["content"]},205 {"role": "user", "content": [{"type": "tool_result", "tool_use_id": tu["id"],206 "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG1x1}}]}]}]},207 beta="computer-use-2025-11-24")208call("h3_computer_20250124_haiku", {"model": HAIKU, "max_tokens": 100,209 "tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, "display_height_px": 768}],210 "messages": [{"role": "user", "content": "Take a screenshot."}]}, beta="computer-use-2025-01-24")211call("h4_computer_toolset_20260801_sonnet5", {"model": SONNET5, "max_tokens": 100, "tools": [{"type": "computer_toolset_20260801", "configs": {"zoom": {"enabled": False}}}],212 "messages": [{"role": "user", "content": "Take a screenshot."}]})213call("h5_computer_toolset_20260801_haiku_unsupported", {"model": HAIKU, "max_tokens": 20, "tools": [{"type": "computer_toolset_20260801"}],214 "messages": [{"role": "user", "content": "Take a screenshot."}]})215call("h6_text_editor_20250728", {"model": HAIKU, "max_tokens": 120, "tools": [{"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}],216 "messages": [{"role": "user", "content": "View the directory /tmp."}]})217call("h7_bash_20250124", {"model": HAIKU, "max_tokens": 120, "tools": [{"type": "bash_20250124", "name": "bash"}],218 "messages": [{"role": "user", "content": "Run `echo OK` in the shell."}]})219call("h8_memory_20250818", {"model": HAIKU, "max_tokens": 150, "tools": [{"type": "memory_20250818", "name": "memory"}],220 "messages": [{"role": "user", "content": "Remember that my favorite color is blue."}]})221call("h9_text_editor_wrong_name_400", {"model": HAIKU, "max_tokens": 20, "tools": [{"type": "text_editor_20250728", "name": "editor"}],222 "messages": [{"role": "user", "content": "Reply OK"}]})223224# ---------------- (i) MCP connector ----------------225MCP = {"model": HAIKU, "max_tokens": 200,226 "mcp_servers": [{"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"}],227 "tools": [{"type": "mcp_toolset", "mcp_server_name": "deepwiki"}],228 "messages": [{"role": "user", "content": "Call the read_wiki_structure tool for the GitHub repo anthropics/anthropic-sdk-python and reply with only the first 3 topic names."}]}229call("i1_mcp_connector_deepwiki", MCP, beta="mcp-client-2025-11-20")230call("i2_mcp_connector_no_beta_header", dict(MCP, max_tokens=20), beta=None)231232# ---------------- (j) count_tokens ----------------233call("j1_count_no_tools", {"model": HAIKU, "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")234call("j2_count_one_tool", {"model": HAIKU, "tools": [WEATHER], "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")235call("j3_count_tool_choice_any", {"model": HAIKU, "tools": [WEATHER], "tool_choice": {"type": "any"}, "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")236call("j4_count_bash", {"model": HAIKU, "tools": [{"type": "bash_20250124", "name": "bash"}], "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")237call("j5_count_text_editor", {"model": HAIKU, "tools": [{"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}], "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")238call("j6_count_memory", {"model": HAIKU, "tools": [{"type": "memory_20250818", "name": "memory"}], "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")239call("j7_count_web_search_400", {"model": HAIKU, "tools": [{"type": "web_search_20250305", "name": "web_search"}], "messages": [{"role": "user", "content": "Weather in Paris?"}]}, path="/v1/messages/count_tokens")240call("j8_count_computer_20250124", {"model": HAIKU, "tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, "display_height_px": 768}],241 "messages": [{"role": "user", "content": "Weather in Paris?"}]}, beta="computer-use-2025-01-24", path="/v1/messages/count_tokens")242243if SUMMARY:244 live.save_sanitized({"total_est_cost_usd": round(TOTAL, 4), "steps": SUMMARY}, OUT / "_summary.json")245 print(f"TOTAL est cost: ${TOTAL:.4f}")246