Python 88.3%
TypeScript 7.6%
Shell 4.1%
1"""Smoke tests for the Anthropic tool-use ecosystem (API Atlas).23Cheap tests (always run, ~$0.005 total on claude-haiku-4-5-20251001): custom tool forced call + tool_result round trip,4parallel/disable_parallel, tool_choice none, count_tokens overhead, Anthropic-schema client tools (bash / text editor /5memory — nothing executed), free 400 validation probes (schema tag list, deferred+cache_control, wrong fixed name).6Expensive tests (RUN_EXPENSIVE_TESTS=true): web search ($0.01/search), web fetch, code execution + container reuse,7tool search, programmatic tool calling (claude-sonnet-4-6), computer use beta (claude-sonnet-4-6), MCP connector.8Reference run: 2026-09-18 (tmp-live/tools/*.json).9"""10from __future__ import annotations1112import json1314import pytest1516HAIKU = "claude-haiku-4-5-20251001"17WEATHER = {"name": "get_weather", "description": "Get the current weather for a city.",18 "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}19TIME = {"name": "get_time", "description": "Get the current local time in a city.",20 "input_schema": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}212223def blocks(body, t):24 return [b for b in body.get("content", []) if b.get("type") == t]252627def _msg(anthropic, body, beta=None, note=""):28 st, out, _ = anthropic("POST", "/v1/messages", body, beta=beta, note=f"test_tools {note}")29 return st, out303132# ----------------------------------------------------------------------------- cheap33def test_forced_tool_then_tool_result(anthropic):34 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 200, "tools": [WEATHER], "tool_choice": {"type": "tool", "name": "get_weather"},35 "messages": [{"role": "user", "content": "Weather in Paris?"}]}, note="forced")36 assert st == 200 and r["stop_reason"] == "tool_use"37 tu = blocks(r, "tool_use")38 assert len(tu) == 1 and tu[0]["name"] == "get_weather" and tu[0]["id"].startswith("toolu_")39 assert "location" in tu[0]["input"] and tu[0].get("caller") == {"type": "direct"}40 st2, r2 = _msg(anthropic, {"model": HAIKU, "max_tokens": 60, "tools": [WEATHER],41 "messages": [{"role": "user", "content": "Weather in Paris?"}, {"role": "assistant", "content": r["content"]},42 {"role": "user", "content": [{"type": "tool_result", "tool_use_id": tu[0]["id"], "content": "18 C, light rain"}]}]}, note="tool_result")43 assert st2 == 200 and r2["stop_reason"] == "end_turn" and blocks(r2, "text")444546def test_disable_parallel_tool_use_limits_to_one(anthropic):47 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 300, "tools": [WEATHER, TIME], "tool_choice": {"type": "any", "disable_parallel_tool_use": True},48 "messages": [{"role": "user", "content": "Weather AND local time in Tokyo, call both tools."}]}, note="disable_parallel")49 assert st == 200 and len(blocks(r, "tool_use")) == 1505152def test_tool_choice_none(anthropic):53 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 30, "tools": [WEATHER], "tool_choice": {"type": "none"},54 "messages": [{"role": "user", "content": "Weather in Paris? Reply in 5 words."}]}, note="none")55 assert st == 200 and not blocks(r, "tool_use") and r["stop_reason"] == "end_turn"565758def test_text_before_tool_result_is_400(anthropic):59 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 20, "tools": [WEATHER],60 "messages": [{"role": "user", "content": "Weather in Paris?"},61 {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_test_01", "name": "get_weather", "input": {"location": "Paris"}}]},62 {"role": "user", "content": [{"type": "text", "text": "Here:"}, {"type": "tool_result", "tool_use_id": "toolu_test_01", "content": "18 C"}]}]}, note="400 ordering")63 assert st == 400 and "tool_result" in r["error"]["message"]646566def test_count_tokens_tool_overhead(anthropic):67 base = {"model": HAIKU, "messages": [{"role": "user", "content": "Weather in Paris?"}]}68 st0, r0, _ = anthropic("POST", "/v1/messages/count_tokens", base, note="test_tools count no tools")69 st1, r1, _ = anthropic("POST", "/v1/messages/count_tokens", {**base, "tools": [WEATHER]}, note="test_tools count one tool")70 st2, r2, _ = anthropic("POST", "/v1/messages/count_tokens", {**base, "tools": [WEATHER], "tool_choice": {"type": "any"}}, note="test_tools count any")71 assert st0 == st1 == st2 == 20072 assert r1["input_tokens"] - r0["input_tokens"] > 300 # tool-use system prompt (~496 on haiku 4.5) + definition73 assert r2["input_tokens"] > r1["input_tokens"] # any/tool prefill costs more than auto74 st3, r3, _ = anthropic("POST", "/v1/messages/count_tokens", {**base, "tools": [{"type": "web_search_20250305", "name": "web_search"}]}, note="test_tools count server tool")75 assert st3 == 400 and "count_tokens" in r3["error"]["message"]767778@pytest.mark.parametrize("tool,expected_cmd_key", [79 ({"type": "bash_20250124", "name": "bash"}, "command"),80 ({"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, "command"),81 ({"type": "memory_20250818", "name": "memory"}, "command"),82])83def test_anthropic_schema_client_tools_emit_tool_use(anthropic, tool, expected_cmd_key):84 prompt = {"bash": "Run `echo OK` in the shell.", "str_replace_based_edit_tool": "View the directory /tmp.", "memory": "Remember that my favorite color is blue."}[tool["name"]]85 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 150, "tools": [tool], "messages": [{"role": "user", "content": prompt}]}, note=tool["type"])86 assert st == 200 and r["stop_reason"] == "tool_use"87 tu = blocks(r, "tool_use")[0]88 assert tu["name"] == tool["name"] and expected_cmd_key in tu["input"]899091def test_fixed_names_and_validation_400s(anthropic):92 ok = [{"role": "user", "content": "Reply with OK."}]93 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 5, "tools": [{"type": "text_editor_20250728", "name": "editor"}], "messages": ok}, note="wrong name")94 assert st == 400 and "str_replace_based_edit_tool" in r["error"]["message"]95 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 5, "tools": [{"type": "atlas_probe_20990101", "name": "x"}], "messages": ok}, note="unknown type")96 assert st == 400 and "web_search_20250305" in r["error"]["message"] # error lists every schema-accepted type tag97 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 5, "tools": [dict(WEATHER, defer_loading=True)], "messages": ok}, note="all deferred")98 assert st == 400 and "defer_loading" in r["error"]["message"]99 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 5, "tools": [{"type": "web_search_20250305", "name": "web_search", "allowed_domains": ["a.com"], "blocked_domains": ["b.com"]}], "messages": ok}, note="both domain lists")100 assert st == 400101 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 5, "mcp_servers": [{"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "d"}],102 "tools": [{"type": "mcp_toolset", "mcp_server_name": "d"}], "messages": ok}, note="mcp no header")103 assert st == 400 and "mcp-client-2025-11-20" in r["error"]["message"]104105106def test_streaming_input_json_delta(anthropic):107 st, lines, _ = anthropic("POST", "/v1/messages", {"model": HAIKU, "max_tokens": 150, "stream": True, "tools": [dict(WEATHER, eager_input_streaming=True)],108 "tool_choice": {"type": "any"}, "messages": [{"role": "user", "content": "Weather in San Francisco, CA?"}]},109 stream=True, note="test_tools stream")110 assert st == 200111 events = [json.loads(ln[6:]) for ln in lines if ln.startswith("data: ")]112 starts = [e for e in events if e["type"] == "content_block_start" and e["content_block"]["type"] == "tool_use"]113 deltas = [e["delta"]["partial_json"] for e in events if e["type"] == "content_block_delta" and e["delta"]["type"] == "input_json_delta"]114 assert starts and starts[0]["content_block"]["input"] == {} and deltas115 assert json.loads("".join(deltas)).get("location")116 assert any(e["type"] == "message_delta" and e["delta"]["stop_reason"] == "tool_use" for e in events)117118119# ----------------------------------------------------------------------------- expensive120@pytest.mark.run_expensive_tests121def test_web_search_blocks(anthropic):122 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 300, "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 1}],123 "messages": [{"role": "user", "content": "You MUST call the web_search tool once: search for the current Bank of Canada policy interest rate. Answer in 5 words with a citation."}]}, note="web_search")124 assert st == 200125 stu = blocks(r, "server_tool_use")126 if not stu:127 pytest.skip("model answered without searching (allowed; c1 did this)")128 assert stu[0]["id"].startswith("srvtoolu_") and stu[0]["name"] == "web_search"129 res = blocks(r, "web_search_tool_result")[0]130 assert res["tool_use_id"] == stu[0]["id"] and (isinstance(res["content"], list) or res["content"]["type"] == "web_search_tool_result_error")131 assert r["usage"]["server_tool_use"]["web_search_requests"] >= 1132133134@pytest.mark.run_expensive_tests135def test_web_fetch_example_com(anthropic):136 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 200, "tools": [{"type": "web_fetch_20250910", "name": "web_fetch", "max_uses": 1, "citations": {"enabled": True}, "max_content_tokens": 2000}],137 "messages": [{"role": "user", "content": "Fetch https://example.com and tell me the page title in 3 words."}]}, note="web_fetch")138 assert st == 200139 res = blocks(r, "web_fetch_tool_result")[0]140 assert res["content"]["type"] == "web_fetch_result" and res["content"]["content"]["type"] == "document"141 assert r["usage"]["server_tool_use"]["web_fetch_requests"] == 1142143144@pytest.mark.run_expensive_tests145def test_code_execution_and_container_reuse(anthropic):146 tools = [{"type": "code_execution_20250825", "name": "code_execution"}]147 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 400, "tools": tools, "messages": [{"role": "user", "content": "Use the code execution tool to run print(2+2) and reply with only the number."}]}, note="code_exec")148 assert st == 200 and r["container"]["id"].startswith("container_") and "expires_at" in r["container"]149 assert blocks(r, "server_tool_use")[0]["name"] == "bash_code_execution"150 assert blocks(r, "bash_code_execution_tool_result")[0]["content"]["type"] == "bash_code_execution_result"151 st2, r2 = _msg(anthropic, {"model": HAIKU, "max_tokens": 300, "container": r["container"]["id"], "tools": tools,152 "messages": [{"role": "user", "content": "Run the shell command `echo hi` with the code execution tool and reply with only its output."}]}, note="container reuse")153 assert st2 == 200 and r2["container"]["id"] == r["container"]["id"]154155156@pytest.mark.run_expensive_tests157def test_tool_search_regex(anthropic):158 schema = {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}159 tools = [{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}] + [160 {"name": n, "description": d, "input_schema": schema, "defer_loading": True}161 for n, d in [("get_weather", "Get the current weather for a city"), ("send_email", "Send an email"), ("translate_text", "Translate text")]]162 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 300, "tools": tools, "messages": [{"role": "user", "content": "What is the weather in Paris? Find and call the right tool."}]}, note="tool_search")163 assert st == 200164 res = blocks(r, "tool_search_tool_result")[0]["content"]165 assert res["type"] == "tool_search_tool_search_result" and {"type": "tool_reference", "tool_name": "get_weather"} in res["tool_references"]166 assert any(b["name"] == "get_weather" for b in blocks(r, "tool_use"))167168169@pytest.mark.run_expensive_tests170def test_programmatic_tool_calling_caller(anthropic):171 tools = [{"type": "code_execution_20260120", "name": "code_execution"},172 {"name": "query_database", "description": "Execute a SQL query. Returns a JSON array of row objects.",173 "input_schema": {"type": "object", "properties": {"sql": {"type": "string"}}, "required": ["sql"]}, "allowed_callers": ["code_execution_20260120"]}]174 st, r = _msg(anthropic, {"model": "claude-sonnet-4-6", "max_tokens": 500, "tools": tools,175 "messages": [{"role": "user", "content": "Use code to call query_database with sql 'SELECT 1 AS one' and tell me the value of one."}]}, note="ptc")176 assert st == 200 and r["container"]["id"]177 prog = [b for b in blocks(r, "tool_use") if b["caller"]["type"] == "code_execution_20260120"]178 assert prog and prog[0]["caller"]["tool_id"].startswith("srvtoolu_")179 st_h, r_h = _msg(anthropic, {"model": HAIKU, "max_tokens": 50, "tools": tools, "messages": [{"role": "user", "content": "Reply with OK."}]}, note="ptc haiku 400")180 assert st_h == 400 and "programmatic tool calling" in r_h["error"]["message"]181182183@pytest.mark.run_expensive_tests184def test_computer_use_beta_screenshot(anthropic):185 cu = {"type": "computer_20251124", "name": "computer", "display_width_px": 1024, "display_height_px": 768}186 st, r = _msg(anthropic, {"model": "claude-sonnet-4-6", "max_tokens": 100, "tools": [cu], "messages": [{"role": "user", "content": "Take a screenshot."}]},187 beta="computer-use-2025-11-24", note="computer_20251124")188 assert st == 200 and blocks(r, "tool_use")[0]["input"]["action"] == "screenshot"189 st_no, _ = _msg(anthropic, {"model": "claude-sonnet-4-6", "max_tokens": 5, "tools": [cu], "messages": [{"role": "user", "content": "Reply with OK."}]}, note="computer no header")190 assert st_no == 400191192193@pytest.mark.run_expensive_tests194def test_mcp_connector_deepwiki(anthropic):195 st, r = _msg(anthropic, {"model": HAIKU, "max_tokens": 200, "mcp_servers": [{"type": "url", "url": "https://mcp.deepwiki.com/mcp", "name": "deepwiki"}],196 "tools": [{"type": "mcp_toolset", "mcp_server_name": "deepwiki"}],197 "messages": [{"role": "user", "content": "Call the read_wiki_structure tool for the GitHub repo anthropics/anthropic-sdk-python and reply with only the first 3 topic names."}]},198 beta="mcp-client-2025-11-20", note="mcp")199 assert st == 200200 mtu = blocks(r, "mcp_tool_use")201 if not mtu:202 pytest.skip("model did not call the MCP tool")203 assert mtu[0]["id"].startswith("mcptoolu_") and mtu[0]["server_name"] == "deepwiki"204 assert blocks(r, "mcp_tool_result")[0]["tool_use_id"] == mtu[0]["id"]205