spb/doc-api

1#!/usr/bin/env python32"""Computer use agent loop skeleton (client-side execution) for the Anthropic Messages API.34STATUS (2026-09-18):5  * PARTS THAT RAN LIVE: the request shape for `computer_20251124` + header `computer-use-2025-11-24` on claude-sonnet-4-66    (h1: Claude returned tool_use {"action":"screenshot"}) and the follow-up turn returning a 1x1 PNG screenshot as an7    image tool_result (h2: Claude described the screen). `computer_toolset_20260801` on claude-sonnet-5 (h4) returned8    tool_use {name:"screenshot", toolset_name:"computer", input:{}}. `computer_20250124` on haiku 4.5 (h3) also ran.9  * PARTS THAT DID NOT RUN: no real display was driven. `execute_action` below is a stub that returns a placeholder PNG for10    screenshot/zoom and "OK" for other actions. Wire it to xdotool / pyautogui / a VM in a sandbox before real use.1112Run (2 requests, ≈ $0.013 on sonnet 4.6):  .venv/bin/python examples/anthropic/computer-use/computer_use_loop.py [--toolset]13  --toolset  use computer_toolset_20260801 (GA, needs claude-sonnet-5 / opus-5 / opus-4-8 / fable-5*) instead of computer_20251124 (beta).14"""15from __future__ import annotations1617import json18import sys19from pathlib import Path2021ROOT = Path(__file__).resolve().parents[3]22sys.path.insert(0, str(ROOT))23from scripts.live import anthropic_request  # noqa: E4022425PLACEHOLDER_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="26NOT_EXECUTED = "Not executed: an earlier computer action in this turn failed."  # exact halt text (toolset batch actions)27USE_TOOLSET = "--toolset" in sys.argv28MODEL = "claude-sonnet-5" if USE_TOOLSET else "claude-sonnet-4-6"29BETA = None if USE_TOOLSET else "computer-use-2025-11-24"30TOOLS = [{"type": "computer_toolset_20260801", "configs": {"zoom": {"enabled": False}}}] if USE_TOOLSET else [31    {"type": "computer_20251124", "name": "computer", "display_width_px": 1024, "display_height_px": 768, "display_number": 1}]32SCALE = 1.0  # screenshot px -> real display px (coordinates come back in screenshot space; multiply by 1/SCALE before clicking)333435def execute_action(action: str, params: dict):36    """STUB — replace with real display automation. Returns str or a list of content blocks."""37    if action in ("screenshot", "zoom"):38        return [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PLACEHOLDER_PNG}}]39    if action == "cursor_position":40        return "X=512, Y=384"41    if action in ("left_click", "right_click", "middle_click", "double_click", "triple_click", "mouse_move", "left_click_drag", "scroll"):42        coord = params.get("coordinate") or params.get("start_coordinate")43        if coord and not (0 <= coord[0] < 1024 and 0 <= coord[1] < 768):44            raise ValueError("Coordinates out of bounds")45        return "OK"  # would call: click(coord[0] / SCALE, coord[1] / SCALE)46    if action in ("type", "key", "hold_key", "wait", "left_mouse_down", "left_mouse_up"):47        return "OK"48    raise ValueError(f"Unknown or unimplemented action: {action}")495051def process_tool_calls(content: list[dict]) -> list[dict]:52    """Run every computer action in order; after the first failure, skip the rest with the halt text (batch actions)."""53    results, failed = [], False54    for block in content:55        if block.get("type") != "tool_use":56            continue57        is_member = block.get("toolset_name") == "computer"58        if not (is_member or block["name"] == "computer"):59            continue  # other tools (bash, text editor, custom) would be dispatched here60        action = block["name"] if is_member else block["input"].get("action")61        params = block["input"]62        res: dict = {"type": "tool_result", "tool_use_id": block["id"]}63        if is_member:64            res["toolset_name"] = "computer"  # mandatory echo for toolset members65        if failed:66            res.update(content=NOT_EXECUTED, is_error=True)67        else:68            try:69                res["content"] = execute_action(action, params)70            except Exception as err:  # noqa: BLE00171                res.update(content=f"Error: {err}", is_error=True)72                failed = True73        results.append(res)74    return results757677def sampling_loop(prompt: str, max_iterations: int = 2, max_tokens: int = 100) -> list[dict]:78    messages = [{"role": "user", "content": prompt}]79    for i in range(max_iterations):80        st, resp, _ = anthropic_request("POST", "/v1/messages", {"model": MODEL, "max_tokens": max_tokens, "tools": TOOLS, "messages": messages},81                                        beta=BETA, note=f"computer_use_loop.py iter {i} {TOOLS[0]['type']}")82        if st != 200:83            raise RuntimeError(json.dumps(resp)[:300])84        messages.append({"role": "assistant", "content": resp["content"]})85        print(f"[{i}] stop_reason={resp['stop_reason']}", [(b.get("name"), b.get("input")) for b in resp["content"] if b["type"] == "tool_use"])86        results = process_tool_calls(resp["content"])87        if not results or resp["stop_reason"] != "tool_use":88            break89        messages.append({"role": "user", "content": results})90        max_tokens = 5091    return messages929394if __name__ == "__main__":95    msgs = sampling_loop("Take a screenshot.")96    print(json.dumps(msgs[-1], indent=1)[:600])97