Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Live probes for the Anthropic advanced-generation domain (API Atlas).3Runs every probe once, saves sanitized raw responses under tmp-live/anthropic-advanced/,4logs each call to reports/live-requests.jsonl with an estimated cost computed from usage.5"""6from __future__ import annotations7import base64, json, os, sys, time, uuid, zlib, struct8from pathlib import Path9ROOT = Path(__file__).resolve().parents[2]10sys.path.insert(0, str(ROOT))11from scripts import live # noqa: E4021213OUT = ROOT / "tmp-live" / "anthropic-advanced"14OUT.mkdir(parents=True, exist_ok=True)15HAIKU = "claude-haiku-4-5-20251001"16SONNET5 = "claude-sonnet-5"17SONNET46 = "claude-sonnet-4-6"18OPUS5 = "claude-opus-5"19# USD per MTok: input, output, 5m write, 1h write, read20PRICES = {21 HAIKU: (1, 5, 1.25, 2, 0.10),22 SONNET5: (2, 10, 2.5, 4, 0.20),23 SONNET46: (3, 15, 3.75, 6, 0.30),24 OPUS5: (5, 25, 6.25, 10, 0.50),25}26SUMMARY: list[dict] = []27TOTAL = [0.0]282930def cost(model: str, usage: dict | None) -> float:31 if not usage or model not in PRICES:32 return 0.033 i, o, w5, w1, r = PRICES[model]34 cc = usage.get("cache_creation") or {}35 w5t = cc.get("ephemeral_5m_input_tokens")36 w1t = cc.get("ephemeral_1h_input_tokens", 0) or 037 if w5t is None:38 w5t = usage.get("cache_creation_input_tokens") or 039 c = (usage.get("input_tokens", 0) * i + usage.get("output_tokens", 0) * o + w5t * w5 + w1t * w140 + (usage.get("cache_read_input_tokens") or 0) * r) / 1e641 if usage.get("speed") == "fast":42 c *= 243 return c444546def call(name: str, method: str, path: str, body=None, *, beta: str | None = None, model: str | None = None,47 stream: bool = False, data: bytes | None = None, content_type: str | None = "application/json",48 note: str = "") -> tuple[int, object, dict]:49 key = os.environ["ANTHROPIC_API_KEY"]50 headers = {"x-api-key": key, "anthropic-version": live.ANTHROPIC_VERSION}51 if content_type:52 headers["Content-Type"] = content_type53 if beta:54 headers["anthropic-beta"] = beta55 if body is not None:56 data = json.dumps(body).encode()57 t0 = time.time()58 st, out, hdrs = live._request(live.ANTHROPIC_BASE + path, method, headers, data, 180, stream)59 events = None60 if stream and st == 200:61 events = [l for l in out if l]62 out = events63 dt = time.time() - t064 usage = None65 if isinstance(out, dict):66 usage = out.get("usage")67 elif isinstance(out, list): # SSE lines68 for l in out:69 if l.startswith("data:"):70 try:71 d = json.loads(l[5:].strip())72 except Exception:73 continue74 if d.get("type") == "message_start":75 usage = dict(d["message"].get("usage") or {})76 elif d.get("type") == "message_delta" and usage is not None:77 usage["output_tokens"] = d.get("usage", {}).get("output_tokens", usage.get("output_tokens", 0))78 mdl = model or (body or {}).get("model") if isinstance(body, dict) else model79 c = cost(mdl or "", usage)80 TOTAL[0] += c81 live.log_request("anthropic", method, path.split("?")[0], st, c,82 f"atlas-anthropic-advanced {name} {mdl or ''} beta={beta or '-'} {note}")83 rec = {"name": name, "method": method, "path": path, "status": st, "model": mdl, "beta": beta,84 "latency_s": round(dt, 2), "est_cost_usd": round(c, 6), "usage": usage,85 "headers": live.interesting_headers(hdrs)}86 if isinstance(out, bytes):87 rec["body_bytes"] = len(out)88 rec["body_preview"] = out[:200].decode("utf-8", "replace")89 else:90 rec["body"] = out91 live.save_sanitized(rec, OUT / f"{name}.json")92 SUMMARY.append({k: rec[k] for k in ("name", "status", "model", "beta", "est_cost_usd", "usage", "latency_s")})93 err = out.get("error", {}).get("message") if isinstance(out, dict) and st >= 400 else ""94 print(f"[{st}] {name} {mdl or ''} ${c:.5f} {dt:.1f}s {err[:160] if err else ''}")95 return st, out, hdrs969798def filler(seed: str, words: int) -> str:99 base = ("The quarterly logistics review covers warehouse throughput, carrier performance, customs "100 "documentation, cold-chain compliance, and regional demand forecasting for the coming period. ")101 parts = []102 n = 0103 k = 0104 while n < words:105 parts.append(f"Section {seed}-{k}: " + base)106 n += len(base.split()) + 2107 k += 1108 return "\n".join(parts)109110111def tiny_png() -> str:112 def chunk(t, d):113 c = struct.pack(">I", len(d)) + t + d114 return c + struct.pack(">I", zlib.crc32(t + d) & 0xffffffff)115 raw = b"\x00\xff\x00\x00" # 1 row: filter 0 + RGB red pixel116 png = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))117 + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))118 return base64.b64encode(png).decode()119120121def tiny_pdf(text: str) -> bytes:122 stream = f"BT /F1 18 Tf 40 750 Td ({text}) Tj ET".encode()123 objs = [124 b"<< /Type /Catalog /Pages 2 0 R >>",125 b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",126 b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>",127 b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream",128 b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",129 ]130 out = b"%PDF-1.4\n"131 offsets = []132 for i, o in enumerate(objs, 1):133 offsets.append(len(out))134 out += f"{i} 0 obj\n".encode() + o + b"\nendobj\n"135 xref = len(out)136 out += f"xref\n0 {len(objs)+1}\n0000000000 65535 f \n".encode()137 for off in offsets:138 out += f"{off:010d} 00000 n \n".encode()139 out += f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()140 return out141142143def multipart(fields: dict, filename: str, content: bytes, mime: str) -> tuple[bytes, str]:144 b = "----atlas" + uuid.uuid4().hex145 body = b""146 for k, v in fields.items():147 body += f"--{b}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n".encode()148 body += (f"--{b}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{filename}\"\r\n"149 f"Content-Type: {mime}\r\n\r\n").encode() + content + f"\r\n--{b}--\r\n".encode()150 return body, f"multipart/form-data; boundary={b}"151152153def msg(model, content, **kw):154 body = {"model": model, "max_tokens": kw.pop("max_tokens", 16),155 "messages": [{"role": "user", "content": content}]}156 body.update(kw)157 return body158159160def run(only: set[str] | None = None):161 def want(tag):162 return only is None or tag in only163164 # ---------------- (a) prompt caching (Haiku minimum is 4096 tokens) ----------------165 if want("cache"):166 sys_text = filler("A", 3600) # ~4.5k tokens167 system = [{"type": "text", "text": sys_text, "cache_control": {"type": "ephemeral"}}]168 b = msg(HAIKU, "Reply with OK.", system=system, max_tokens=8)169 call("cache_01_write", "POST", "/v1/messages", b)170 call("cache_02_read", "POST", "/v1/messages", b)171 # automatic (top-level) cache_control, same system prefix -> expect read of system + write of tail172 b_auto = msg(HAIKU, "Reply with OK.", system=[{"type": "text", "text": sys_text}], max_tokens=8,173 cache_control={"type": "ephemeral"})174 call("cache_03_auto_top_level", "POST", "/v1/messages", b_auto)175 # 1h TTL on a fresh prefix176 sys1h = [{"type": "text", "text": filler("B", 3600), "cache_control": {"type": "ephemeral", "ttl": "1h"}}]177 call("cache_04_ttl_1h_write", "POST", "/v1/messages", msg(HAIKU, "Reply with OK.", system=sys1h, max_tokens=8))178 # below minimum: ~600 tokens179 small = [{"type": "text", "text": filler("C", 450), "cache_control": {"type": "ephemeral"}}]180 call("cache_05_below_minimum", "POST", "/v1/messages", msg(HAIKU, "Reply with OK.", system=small, max_tokens=8))181 # caching on tools[] block182 tools = [{"name": f"tool_{i}", "description": filler(f"T{i}", 700),183 "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}} for i in range(6)]184 tools[-1]["cache_control"] = {"type": "ephemeral"}185 bt = msg(HAIKU, "Reply with OK. Do not call tools.", tools=tools, max_tokens=8)186 call("cache_06_tools_write", "POST", "/v1/messages", bt)187 call("cache_07_tools_read", "POST", "/v1/messages", bt)188 # pre-warm with max_tokens: 0189 call("cache_08_prewarm_max_tokens_0", "POST", "/v1/messages", msg(HAIKU, "warmup", system=system, max_tokens=0))190 # invalid ttl value191 bad = [{"type": "text", "text": sys_text, "cache_control": {"type": "ephemeral", "ttl": "2h"}}]192 call("cache_09_invalid_ttl", "POST", "/v1/messages", msg(HAIKU, "Reply with OK.", system=bad, max_tokens=8))193 # 5 breakpoints -> error?194 five = [{"type": "text", "text": f"Block {i}.", "cache_control": {"type": "ephemeral"}} for i in range(5)]195 call("cache_10_five_breakpoints", "POST", "/v1/messages", msg(HAIKU, "Reply with OK.", system=five, max_tokens=8))196197 # ---------------- (b) extended thinking on Haiku ----------------198 if want("thinking"):199 th = {"type": "enabled", "budget_tokens": 1024}200 call("think_01_enabled", "POST", "/v1/messages",201 msg(HAIKU, "What is 2+2? Reply with the number.", thinking=th, max_tokens=1200))202 call("think_02_enabled_stream", "POST", "/v1/messages",203 msg(HAIKU, "What is 2+2? Reply with the number.", thinking=th, max_tokens=1200, stream=True), stream=True)204 call("think_03_temperature_conflict", "POST", "/v1/messages",205 msg(HAIKU, "What is 2+2?", thinking=th, max_tokens=1200, temperature=0.5))206 call("think_04_top_k_conflict", "POST", "/v1/messages",207 msg(HAIKU, "What is 2+2?", thinking=th, max_tokens=1200, top_k=5))208 call("think_05_budget_too_small", "POST", "/v1/messages",209 msg(HAIKU, "What is 2+2?", thinking={"type": "enabled", "budget_tokens": 512}, max_tokens=1200))210 call("think_06_budget_ge_max_tokens", "POST", "/v1/messages",211 msg(HAIKU, "What is 2+2?", thinking={"type": "enabled", "budget_tokens": 2000}, max_tokens=1200))212 call("think_07_adaptive_on_haiku", "POST", "/v1/messages",213 msg(HAIKU, "What is 2+2?", thinking={"type": "adaptive"}, max_tokens=1200))214 call("think_08_display_omitted", "POST", "/v1/messages",215 msg(HAIKU, "What is 2+2? Reply with the number.", thinking={**th, "display": "omitted"}, max_tokens=1200))216 call("think_09_forced_tool_choice_conflict", "POST", "/v1/messages",217 msg(HAIKU, "Call the tool.", thinking=th, max_tokens=1200, tool_choice={"type": "any"},218 tools=[{"name": "get_time", "description": "Returns the current time.",219 "input_schema": {"type": "object", "properties": {}}}]))220 # tool-use round trip preserving thinking block221 tool = {"name": "get_time", "description": "Returns the current UTC time.",222 "input_schema": {"type": "object", "properties": {}}}223 st, r1, _ = call("think_10_tool_turn1", "POST", "/v1/messages",224 msg(HAIKU, "Use the get_time tool, then tell me the time in one short sentence.",225 thinking=th, tools=[tool], max_tokens=1500))226 if st == 200 and isinstance(r1, dict):227 tu = [c for c in r1["content"] if c["type"] == "tool_use"]228 if tu:229 msgs = [{"role": "user", "content": "Use the get_time tool, then tell me the time in one short sentence."},230 {"role": "assistant", "content": r1["content"]},231 {"role": "user", "content": [{"type": "tool_result", "tool_use_id": tu[0]["id"], "content": "12:00:00 UTC"}]}]232 call("think_11_tool_turn2_pass_back", "POST", "/v1/messages",233 {"model": HAIKU, "max_tokens": 1500, "thinking": th, "tools": [tool], "messages": msgs})234 # tampered thinking block235 tampered = json.loads(json.dumps(msgs))236 for c in tampered[1]["content"]:237 if c["type"] == "thinking":238 c["thinking"] = c["thinking"] + " (edited)"239 call("think_12_tool_turn2_tampered", "POST", "/v1/messages",240 {"model": HAIKU, "max_tokens": 1500, "thinking": th, "tools": [tool], "messages": tampered})241 # thinking block dropped -> ?242 dropped = json.loads(json.dumps(msgs))243 dropped[1]["content"] = [c for c in dropped[1]["content"] if c["type"] != "thinking"]244 call("think_13_tool_turn2_thinking_dropped", "POST", "/v1/messages",245 {"model": HAIKU, "max_tokens": 1500, "thinking": th, "tools": [tool], "messages": dropped})246247 # ---------------- (c) adaptive / interleaved ----------------248 if want("adaptive"):249 call("adaptive_01_sonnet5", "POST", "/v1/messages",250 msg(SONNET5, "What is 2+2? Reply with the number.", thinking={"type": "adaptive", "display": "summarized"}, max_tokens=600))251 call("adaptive_02_sonnet5_disabled", "POST", "/v1/messages",252 msg(SONNET5, "Reply with OK.", thinking={"type": "disabled"}, max_tokens=8))253 call("adaptive_03_sonnet5_enabled_rejected", "POST", "/v1/messages",254 msg(SONNET5, "Reply with OK.", thinking={"type": "enabled", "budget_tokens": 1024}, max_tokens=1200))255 call("adaptive_04_sonnet5_temperature", "POST", "/v1/messages",256 msg(SONNET5, "Reply with OK.", max_tokens=8, temperature=0.2))257 call("adaptive_05_display_updates_no_header", "POST", "/v1/messages",258 msg(SONNET5, "Reply with OK.", thinking={"type": "adaptive", "display": "updates"}, max_tokens=8))259 tool = {"name": "get_time", "description": "Returns the current UTC time.",260 "input_schema": {"type": "object", "properties": {}}}261 call("interleaved_01_sonnet46_enabled_header", "POST", "/v1/messages",262 msg(SONNET46, "Reply with OK. Do not call tools.", thinking={"type": "enabled", "budget_tokens": 1024},263 tools=[tool], max_tokens=1200), beta="interleaved-thinking-2025-05-14")264 call("interleaved_02_haiku_header_ignored", "POST", "/v1/messages",265 msg(HAIKU, "Reply with OK. Do not call tools.", thinking={"type": "enabled", "budget_tokens": 1024},266 tools=[tool], max_tokens=1200), beta="interleaved-thinking-2025-05-14")267268 # ---------------- (d) effort ----------------269 if want("effort"):270 for lvl in ("low", "max", "xhigh"):271 call(f"effort_{lvl}_sonnet5", "POST", "/v1/messages",272 msg(SONNET5, "Reply with OK.", output_config={"effort": lvl}, max_tokens=64))273 call("effort_invalid_sonnet5", "POST", "/v1/messages",274 msg(SONNET5, "Reply with OK.", output_config={"effort": "ultra"}, max_tokens=8))275 call("effort_low_haiku_unsupported", "POST", "/v1/messages",276 msg(HAIKU, "Reply with OK.", output_config={"effort": "low"}, max_tokens=8))277 call("effort_disabled_thinking_max_sonnet5", "POST", "/v1/messages",278 msg(SONNET5, "Reply with OK.", output_config={"effort": "max"}, thinking={"type": "disabled"}, max_tokens=8))279 call("effort_per_message_sonnet5_beta", "POST", "/v1/messages",280 {"model": SONNET5, "max_tokens": 8, "messages": [281 {"role": "system", "content": [], "output_config": {"effort": "low"}},282 {"role": "user", "content": "Reply with OK."}]}, beta="mid-conversation-output-config-2026-07-01")283 call("task_budget_sonnet5_unsupported", "POST", "/v1/messages",284 msg(SONNET5, "Reply with OK.", output_config={"task_budget": {"type": "tokens", "total": 20000}}, max_tokens=8),285 beta="task-budgets-2026-03-13")286287 # ---------------- (e) fast mode ----------------288 if want("fast"):289 call("fast_01_opus5_header", "POST", "/v1/messages",290 msg(OPUS5, "Reply with OK.", speed="fast", max_tokens=8, thinking={"type": "disabled"}), beta="fast-mode-2026-02-01")291 call("fast_02_opus5_no_header", "POST", "/v1/messages",292 msg(OPUS5, "Reply with OK.", speed="fast", max_tokens=8, thinking={"type": "disabled"}))293 call("fast_03_haiku_unsupported", "POST", "/v1/messages",294 msg(HAIKU, "Reply with OK.", speed="fast", max_tokens=8), beta="fast-mode-2026-02-01")295 call("fast_04_opus5_speed_standard", "POST", "/v1/messages",296 msg(OPUS5, "Reply with OK.", speed="standard", max_tokens=8, thinking={"type": "disabled"}), beta="fast-mode-2026-02-01")297298 # ---------------- (f) 1M context header ----------------299 if want("ctx1m"):300 call("ctx1m_01_sonnet5_header", "POST", "/v1/messages", msg(SONNET5, "Reply with OK.", max_tokens=8),301 beta="context-1m-2025-08-07")302 call("ctx1m_02_haiku_header", "POST", "/v1/messages", msg(HAIKU, "Reply with OK.", max_tokens=8),303 beta="context-1m-2025-08-07")304305 # ---------------- (g) context editing / compaction ----------------306 if want("ctxmgmt"):307 tool = {"name": "lookup", "description": "Looks up a record.",308 "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}}}309 hist = [{"role": "user", "content": "Look up records r1 and r2 then reply OK."}]310 for i in range(3):311 hist.append({"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_atlas{i}", "name": "lookup", "input": {"id": f"r{i}"}}]})312 hist.append({"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"toolu_atlas{i}", "content": "record " + ("x" * 400)}]})313 hist.append({"role": "user", "content": "Reply with OK. Do not call tools."})314 # consecutive user messages are merged by the API; keep as is315 cm = {"edits": [{"type": "clear_tool_uses_20250919", "trigger": {"type": "input_tokens", "value": 1},316 "keep": {"type": "tool_uses", "value": 1}, "clear_at_least": {"type": "input_tokens", "value": 1}}]}317 call("ctxedit_01_clear_tool_uses", "POST", "/v1/messages",318 {"model": HAIKU, "max_tokens": 8, "tools": [tool], "messages": hist, "context_management": cm},319 beta="context-management-2025-06-27")320 call("ctxedit_02_count_tokens", "POST", "/v1/messages/count_tokens",321 {"model": HAIKU, "tools": [tool], "messages": hist, "context_management": cm}, beta="context-management-2025-06-27")322 call("ctxedit_03_no_header", "POST", "/v1/messages",323 {"model": HAIKU, "max_tokens": 8, "tools": [tool], "messages": hist, "context_management": cm})324 call("ctxedit_04_clear_thinking_haiku", "POST", "/v1/messages",325 msg(HAIKU, "Reply with OK.", max_tokens=8,326 context_management={"edits": [{"type": "clear_thinking_20251015", "keep": {"type": "thinking_turns", "value": 1}}]}),327 beta="context-management-2025-06-27")328 comp = {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50000}}]}329 call("compact_01_sonnet5_accepted", "POST", "/v1/messages",330 msg(SONNET5, "Reply with OK.", max_tokens=8, context_management=comp), beta="compact-2026-01-12")331 call("compact_02_haiku_unsupported", "POST", "/v1/messages",332 msg(HAIKU, "Reply with OK.", max_tokens=8, context_management=comp), beta="compact-2026-01-12")333 call("compact_03_trigger_too_low", "POST", "/v1/messages",334 msg(SONNET5, "Reply with OK.", max_tokens=8,335 context_management={"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 1000}}]}),336 beta="compact-2026-01-12")337 call("compact_04_on_demand_summarize", "POST", "/v1/messages",338 {"model": SONNET5, "max_tokens": 400, "compaction": {"type": "summarize"},339 "messages": [{"role": "user", "content": "My name is Ada and I like teal."},340 {"role": "assistant", "content": "Nice to meet you, Ada."}]}, beta="compact-2026-09-04")341342 # ---------------- (h) structured outputs ----------------343 if want("structured"):344 schema = {"type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"},345 "city": {"type": "string"}}, "required": ["name", "age", "city"], "additionalProperties": False}346 fmt = {"type": "json_schema", "schema": schema}347 prompt = "Ada is 36 and lives in Lyon. Extract the person."348 call("so_01_json_schema", "POST", "/v1/messages", msg(HAIKU, prompt, output_config={"format": fmt}, max_tokens=100))349 call("so_02_json_schema_stream", "POST", "/v1/messages",350 msg(HAIKU, prompt, output_config={"format": fmt}, max_tokens=100, stream=True), stream=True)351 bad = {"type": "object", "patternProperties": {"^x-": {"type": "string"}}, "additionalProperties": False}352 call("so_03_unsupported_patternProperties", "POST", "/v1/messages",353 msg(HAIKU, prompt, output_config={"format": {"type": "json_schema", "schema": bad}}, max_tokens=50))354 bad2 = {"type": "object", "properties": {"age": {"type": "integer", "minimum": 0}}, "required": ["age"], "additionalProperties": False}355 call("so_04_unsupported_minimum", "POST", "/v1/messages",356 msg(HAIKU, prompt, output_config={"format": {"type": "json_schema", "schema": bad2}}, max_tokens=50))357 call("so_05_count_tokens", "POST", "/v1/messages/count_tokens",358 {"model": HAIKU, "messages": [{"role": "user", "content": prompt}], "output_config": {"format": fmt}})359 call("so_06_with_citations_conflict", "POST", "/v1/messages",360 msg(HAIKU, [{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Ada is 36."},361 "citations": {"enabled": True}}, {"type": "text", "text": prompt}],362 output_config={"format": fmt}, max_tokens=50))363 call("so_07_legacy_output_format", "POST", "/v1/messages",364 msg(HAIKU, prompt, output_format=fmt, max_tokens=100))365 call("so_08_strict_tool", "POST", "/v1/messages",366 msg(HAIKU, "Record Ada, 36, Lyon with the tool.", max_tokens=200,367 tools=[{"name": "record_person", "description": "Records a person.", "strict": True, "input_schema": schema}],368 tool_choice={"type": "tool", "name": "record_person"}))369370 # ---------------- (i) citations ----------------371 if want("citations"):372 doc = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "The sky is blue. Grass is green."},373 "title": "Colors", "citations": {"enabled": True}}374 q = {"type": "text", "text": "What color is grass? Answer in one sentence and cite."}375 call("cite_01_text_char_location", "POST", "/v1/messages", msg(HAIKU, [doc, q], max_tokens=100))376 sr = {"type": "search_result", "source": "kb://colors/1", "title": "Color facts",377 "content": [{"type": "text", "text": "The sky is blue."}, {"type": "text", "text": "Grass is green."}],378 "citations": {"enabled": True}}379 call("cite_02_search_result_location", "POST", "/v1/messages", msg(HAIKU, [sr, q], max_tokens=100))380 custom = {"type": "document", "source": {"type": "content", "content": [{"type": "text", "text": "The sky is blue."},381 {"type": "text", "text": "Grass is green."}]}, "title": "Colors", "citations": {"enabled": True}}382 call("cite_03_content_block_location", "POST", "/v1/messages", msg(HAIKU, [custom, q], max_tokens=100))383 pdf = tiny_pdf("The sky is blue. Grass is green.")384 (OUT / "tiny.pdf").write_bytes(pdf)385 pdoc = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf",386 "data": base64.b64encode(pdf).decode()}, "title": "Colors PDF", "citations": {"enabled": True}}387 call("cite_04_pdf_page_location", "POST", "/v1/messages", msg(HAIKU, [pdoc, q], max_tokens=100))388 call("cite_05_stream_citations_delta", "POST", "/v1/messages", msg(HAIKU, [doc, q], max_tokens=100, stream=True), stream=True)389 doc2 = dict(doc); doc2 = {**doc, "citations": {"enabled": False}}390 call("cite_06_mixed_enabled_error", "POST", "/v1/messages", msg(HAIKU, [doc, doc2, q], max_tokens=50))391392 # ---------------- (j) vision ----------------393 if want("vision"):394 img = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": tiny_png()}}395 call("vision_01_base64_1x1", "POST", "/v1/messages",396 msg(HAIKU, [img, {"type": "text", "text": "What color is this pixel? One word."}], max_tokens=10))397 url_img = {"type": "image", "source": {"type": "url", "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/280px-PNG_transparency_demonstration_1.png"}}398 call("vision_02_url", "POST", "/v1/messages",399 msg(HAIKU, [url_img, {"type": "text", "text": "Describe this image in five words."}], max_tokens=20))400 bad = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": base64.b64encode(b"notapng").decode()}}401 call("vision_03_invalid_image_data", "POST", "/v1/messages",402 msg(HAIKU, [bad, {"type": "text", "text": "Describe."}], max_tokens=10))403 call("vision_04_count_tokens_image", "POST", "/v1/messages/count_tokens",404 {"model": HAIKU, "messages": [{"role": "user", "content": [img, {"type": "text", "text": "Describe."}]}]})405 txtdoc = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Grass is green."}}406 call("vision_05_text_document_no_citations", "POST", "/v1/messages",407 msg(HAIKU, [txtdoc, {"type": "text", "text": "What color is grass? One word."}], max_tokens=10))408 img_err = {**img, "transformations": {"oversized_image": "error"}}409 call("vision_06_transformations_field", "POST", "/v1/messages",410 msg(HAIKU, [img_err, {"type": "text", "text": "What color? One word."}], max_tokens=10))411412 # ---------------- (k) Files API ----------------413 if want("files"):414 txt = b"The sky is blue. Grass is green. This is a 100-byte-ish plain text file for the API Atlas Files test."415 body, ct = multipart({}, "atlas.txt", txt, "text/plain")416 st, up, _ = call("files_01_upload_txt", "POST", "/v1/files", data=body, content_type=ct, model=HAIKU)417 fid = up.get("id") if isinstance(up, dict) else None418 call("files_02_list", "GET", "/v1/files?limit=2", model=HAIKU)419 call("files_03_list_legacy_beta_header", "GET", "/v1/files?limit=2", beta="files-api-2025-04-14", model=HAIKU)420 if fid:421 call("files_04_list_ids_filter", "GET", f"/v1/files?ids[]={fid}", model=HAIKU)422 call("files_05_retrieve", "GET", f"/v1/files/{fid}", model=HAIKU)423 call("files_06_content_download", "GET", f"/v1/files/{fid}/content", model=HAIKU)424 fdoc = {"type": "document", "source": {"type": "file", "file_id": fid}, "citations": {"enabled": True}}425 call("files_07_message_document_file_id", "POST", "/v1/messages",426 msg(HAIKU, [fdoc, {"type": "text", "text": "What color is grass? One sentence, cite."}], max_tokens=80))427 call("files_08_message_document_file_id_beta_header", "POST", "/v1/messages",428 msg(HAIKU, [{"type": "document", "source": {"type": "file", "file_id": fid}},429 {"type": "text", "text": "What color is grass? One word."}], max_tokens=10),430 beta="files-api-2025-04-14")431 pdf = tiny_pdf("The sky is blue. Grass is green.")432 body, ct = multipart({"expires_in_seconds": "3600"}, "atlas.pdf", pdf, "application/pdf")433 st, up2, _ = call("files_09_upload_pdf_expires", "POST", "/v1/files", data=body, content_type=ct, model=HAIKU)434 pid = up2.get("id") if isinstance(up2, dict) else None435 if pid:436 call("files_10_message_pdf_file_id", "POST", "/v1/messages",437 msg(HAIKU, [{"type": "document", "source": {"type": "file", "file_id": pid}},438 {"type": "text", "text": "What color is grass? One word."}], max_tokens=10))439 call("files_11_image_block_with_pdf_file_error", "POST", "/v1/messages",440 msg(HAIKU, [{"type": "image", "source": {"type": "file", "file_id": pid}},441 {"type": "text", "text": "Describe."}], max_tokens=10))442 png = base64.b64decode(tiny_png())443 body, ct = multipart({}, "pixel.png", png, "image/png")444 st, up3, _ = call("files_12_upload_png", "POST", "/v1/files", data=body, content_type=ct, model=HAIKU)445 iid = up3.get("id") if isinstance(up3, dict) else None446 if iid:447 call("files_13_message_image_file_id", "POST", "/v1/messages",448 msg(HAIKU, [{"type": "image", "source": {"type": "file", "file_id": iid}},449 {"type": "text", "text": "What color? One word."}], max_tokens=10))450 body, ct = multipart({"expires_in_seconds": "60"}, "short.txt", b"x", "text/plain")451 call("files_14_upload_expires_too_short", "POST", "/v1/files", data=body, content_type=ct, model=HAIKU)452 for i, f in enumerate([fid, pid, iid]):453 if f:454 call(f"files_15_delete_{i}", "DELETE", f"/v1/files/{f}", model=HAIKU)455 if fid:456 call("files_16_retrieve_deleted", "GET", f"/v1/files/{fid}", model=HAIKU)457458 # ---------------- extra probes ----------------459 if want("extra"):460 url_img = {"type": "image", "source": {"type": "url", "url": "https://httpbin.org/image/png"}}461 call("vision_07_url_httpbin", "POST", "/v1/messages",462 msg(HAIKU, [url_img, {"type": "text", "text": "Describe this image in five words."}], max_tokens=20))463 call("cache_11_message_level_cache_control", "POST", "/v1/messages",464 {"model": HAIKU, "max_tokens": 8, "messages": [{"role": "user", "content": "Reply with OK.", "cache_control": {"type": "ephemeral"}}]})465 call("think_14_prefill_conflict", "POST", "/v1/messages",466 {"model": HAIKU, "max_tokens": 1200, "thinking": {"type": "enabled", "budget_tokens": 1024},467 "messages": [{"role": "user", "content": "What is 2+2?"}, {"role": "assistant", "content": "The answer is"}]})468 call("adaptive_06_display_updates_with_header", "POST", "/v1/messages",469 msg(SONNET5, "Reply with OK.", thinking={"type": "adaptive", "display": "updates"}, max_tokens=8),470 beta="thinking-display-updates-2026-08-18")471 call("adaptive_07_block_binding_beta", "POST", "/v1/messages",472 msg(SONNET5, "Reply with OK.", thinking={"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}}, max_tokens=8),473 beta="thinking-binding-controls-2026-08-01")474 prev = json.load(open(OUT / "compact_04_on_demand_summarize.json"))["body"]475 if isinstance(prev, dict) and prev.get("content"):476 blk = prev["content"][0]477 call("compact_05_continue_from_summary", "POST", "/v1/messages",478 {"model": SONNET5, "max_tokens": 20, "messages": [479 {"role": "assistant", "content": [blk]},480 {"role": "user", "content": "What is my name and favourite color? Answer in five words."}]},481 beta="compact-2026-09-04")482 call("compact_06_block_misplaced", "POST", "/v1/messages",483 {"model": SONNET5, "max_tokens": 20, "messages": [484 {"role": "user", "content": "My name is Ada and I like teal."},485 {"role": "assistant", "content": [blk]},486 {"role": "user", "content": "What is my name?"}]},487 beta="compact-2026-09-04")488 call("cache_12_diagnostics_beta", "POST", "/v1/messages",489 msg(HAIKU, "Reply with OK.", max_tokens=8, diagnostics={"previous_message_id": None}), beta="cache-diagnosis-2026-04-07")490 b17, ct17 = multipart({}, "plain.txt", b"hello atlas", "application/octet-stream")491 st, up17, _ = call("files_17_upload_octet_stream_part", "POST", "/v1/files", data=b17, content_type=ct17, model=HAIKU)492 if isinstance(up17, dict) and up17.get("id"):493 call("files_18_delete_octet", "DELETE", f"/v1/files/{up17['id']}", model=HAIKU)494495 tag = "-".join(sorted(only)) if only else "all"496 live.save_sanitized({"total_est_cost_usd": round(TOTAL[0], 5), "calls": SUMMARY}, OUT / f"_summary_{tag}.json")497 print(f"\nTOTAL est cost: ${TOTAL[0]:.4f} over {len(SUMMARY)} calls")498499500if __name__ == "__main__":501 run(set(sys.argv[1:]) or None)502