Python 88.3%
TypeScript 7.6%
Shell 4.1%
1"""Offline tests for examples/shared/feature-detection/supports.py using tests/shared/fixtures/{models,tools}.json."""2from __future__ import annotations34import json5import sys6from pathlib import Path78ROOT = Path(__file__).resolve().parents[2]9sys.path.insert(0, str(ROOT / "examples" / "shared" / "feature-detection"))10FIX = ROOT / "tests" / "shared" / "fixtures"1112from supports import Registry, UNKNOWN # noqa: E402131415def reg() -> Registry:16 return Registry.load(FIX / "models.json", FIX / "tools.json")171819def test_tri_state_values():20 r = reg()21 assert r.supports("gpt-5.4-nano", "function_calling") is True22 assert r.supports("gpt-5.4-nano", "fine_tuning") is False23 assert r.supports("gpt-5.4-nano", "extended_thinking") == UNKNOWN # explicit "unknown"24 assert r.supports("gpt-5.4-nano", "never_heard_of_it") == UNKNOWN # missing key25 assert r.supports("ada", "function_calling") == UNKNOWN # capabilities object without booleans26 assert r.supports("does-not-exist", "streaming") == UNKNOWN # unknown model272829def test_alias_and_snapshot_resolution():30 r = reg()31 assert r.canonical_id("claude-haiku-4-5") == "claude-haiku-4-5-20251001"32 assert r.supports("claude-haiku-4-5", "extended_thinking") is True33 assert r.canonical_id("gpt-5.4-nano-2026-03-17") == "gpt-5.4-nano" # snapshot listed + snapshot-kind record defers34 assert r.canonical_id("gpt-5.4-nano-latest") == "gpt-5.4-nano" # alias35 assert r.canonical_id("GPT-5.4-NANO") == "gpt-5.4-nano" # case-insensitive36 assert r.canonical_id("openai/gpt-5.4-nano") == "gpt-5.4-nano" # provider prefix37 assert r.canonical_id("claude-haiku-4-5-20251001-20260101") == "claude-haiku-4-5-20251001" # unlisted date suffix38 assert r.canonical_id("nope") is None394041def test_models_with():42 r = reg()43 both = r.models_with("function_calling", "structured_outputs")44 # deprecated opus excluded, snapshot excluded, xAI retired_redirect + Gemini alias-kind records excluded, voice model (structured_outputs false) excluded45 assert both == ["gpt-5.4-nano", "claude-haiku-4-5-20251001", "grok-4.3", "gemini-3.8-flash", "gemini-3.5-flash-lite"]46 assert r.models_with("function_calling", provider="anthropic") == ["claude-haiku-4-5-20251001"]47 assert r.models_with("extended_thinking", provider="anthropic") == ["claude-haiku-4-5-20251001"]48 assert r.models_with("extended_thinking", include_unknown=True, provider="openai") == ["gpt-5.4-nano"]49 assert "claude-opus-4-1-20250805" in r.models_with("function_calling", statuses_excluded=())50 assert "grok-3" not in r.models_with("function_calling", statuses_excluded=()) # pointer records never listed515253def test_tools_for_and_supports_tool():54 r = reg()55 tools = r.tools_for("claude-haiku-4-5")56 assert "web_fetch_20250910" in tools and "mcp_toolset" in tools57 assert r.supports_tool("claude-haiku-4-5", "web_fetch") is True # name matches versioned type58 assert r.supports_tool("claude-haiku-4-5", "web_search_20250305") is True59 assert r.supports_tool("gpt-5.4-nano", "web_search") is True60 assert r.supports_tool("gpt-5.4-nano", "computer_use_preview") is False # explicit tools[] list, not present61 assert r.supports_tool("ada", "web_search") == UNKNOWN # empty tools[], no tool record62 assert r.supports("gpt-5.4-nano", "tool_computer_use") is False63 assert r.supports("claude-haiku-4-5", "tool_code_execution") == UNKNOWN # explicit "unknown" flag wins over inference64 assert r.supports("claude-opus-4-1", "tool_web_search") is True # inferred from tools[]65 assert r.supports_tool("claude-opus-4-1", "web_fetch") is False # tool record lists compatible models, not this one666768def test_missing_or_malformed_inputs_are_graceful(tmp_path):69 r = Registry.load(tmp_path / "missing.json", None)70 assert r.models == [] and r.supports("x", "y") == UNKNOWN and r.tools_for("x") == []71 bad = tmp_path / "bad.json"72 bad.write_text("{not json")73 assert Registry.load(bad, bad).models == []747576def test_alternative_document_shapes():77 recs = json.loads((FIX / "models.json").read_text())78 wrapped = Registry.from_documents({"records": recs}, {"tools": json.loads((FIX / "tools.json").read_text())})79 assert wrapped.supports("gpt-5.4-nano", "streaming") is True and len(wrapped.tools) == 980 keyed = Registry.from_documents({m["id"]: {k: v for k, v in m.items() if k != "id"} for m in recs})81 assert keyed.supports("claude-haiku-4-5", "prompt_caching") is True828384def test_compare():85 r = reg()86 c = r.compare("gpt-5.4-nano", "claude-haiku-4-5")87 assert c["tool_computer_use"] == (False, True) and c["extended_thinking"] == (UNKNOWN, True)888990# ---------------- xAI records (aliases, retired redirects, parenthetical alias notes) ----------------9192def test_xai_alias_and_retired_redirect_resolution():93 r = reg()94 assert r.canonical_id("grok-4.3-latest") == "grok-4.3" and r.canonical_id("xai/grok-4.3") == "grok-4.3"95 assert r.canonical_id("grok-3") == "grok-4.3" # kind: retired_redirect → target parsed from verification.request_note96 assert r.supports("grok-3", "structured_outputs") is True # queries on the retired id answer for the live target97 assert r.canonical_id("grok-voice-latest") == "grok-voice-think-fast-2.0" # "grok-voice-latest (routes here since …)" note stripped98 assert Registry.redirect_target({"kind": "retired_redirect", "verification": {"request_note": "GET /v1/models/x -> 200 with id 'grok-4.3'"}}) == "grok-4.3"99 assert Registry.redirect_target({"kind": "model"}) is None100101102def test_xai_capabilities_unknown_server_tools_and_tool_inference():103 r = reg()104 assert r.supports("grok-4.3", "function_calling") is True and r.supports("grok-4.3", "structured_outputs") is True105 assert r.supports("grok-4.3", "web_search") == UNKNOWN # explicit "unknown" flag wins over the tools[] entry106 assert r.supports_tool("grok-4.3", "web_search") is True # but the tool list itself says the type string is accepted107 assert r.supports("grok-4.3", "prompt_caching") is True # synonym prompt_caching_automatic108 assert r.supports("grok-4.3", "reasoning") is True and r.supports("grok-4.3", "fine_tuning") is False109 assert r.supports_tool("grok-4.3", "image_generation") is False # explicit list, absent110 assert "mcp" in r.tools_for("grok-4.3") and r.tool_record("xai", "web_search")["category"] == "server"111112113# ---------------- Gemini records (models/ prefix, -latest dict aliases, dict-shaped tools[], synonyms) ----------------114115def test_gemini_prefix_and_latest_alias_resolution():116 r = reg()117 assert r.canonical_id("models/gemini-3.8-flash") == "gemini-3.8-flash"118 assert r.canonical_id("gemini/models/gemini-3.8-flash") == "gemini-3.8-flash"119 assert r.canonical_id("gemini-flash-latest") == "gemini-3.8-flash" # alias record → resolves_to_live (live 2026-09-19)120 assert r.canonical_id("models/gemini-flash-latest") == "gemini-3.8-flash"121 assert r.supports("gemini-flash-latest", "function_calling") is True # answered by the target record, not the alias stub122123124def test_gemini_dict_tools_and_camel_snake_matching():125 r = reg()126 assert r.supports_tool("gemini-3.8-flash", "google_search") is True and r.supports_tool("gemini-3.8-flash", "googleSearch") is True127 assert r.supports_tool("gemini-3.8-flash", "file_search") is False # support: false in the dict entry128 assert r.supports_tool("gemini-3.8-flash", "computer_use") is True # "Supported (Preview)" string → True129 assert r.supports_tool("gemini-3.8-flash", "google_maps") is False # explicit list, absent130 tools = r.tools_for("gemini-3.8-flash")131 assert "google_search" in tools and "file_search" not in tools and "urlContext" not in tools and "url_context" in tools # tool record deduped by normalised key132 assert r.supports("gemini-3.8-flash", "tool_code_execution") is True and r.supports("gemini-3.8-flash", "computer_use") == UNKNOWN # explicit flag wins133134135def test_cross_provider_synonyms_in_models_with():136 r = reg()137 assert r.supports("gemini-3.8-flash", "structured_outputs") is True # ← structured_output138 assert r.supports("gemini-3.8-flash", "web_search") is True # ← google_search_grounding139 assert r.supports("gemini-3.8-flash", "prompt_caching") is True # ← context_caching_implicit140 assert r.models_with("web_search", "structured_outputs", "prompt_caching") == ["gpt-5.4-nano", "claude-haiku-4-5-20251001", "gemini-3.8-flash", "gemini-3.5-flash-lite"]141 # grok-4.3 is excluded because its web_search flag is an explicit "unknown" (include_unknown lets it through)142 assert "grok-4.3" in r.models_with("web_search", "structured_outputs", "prompt_caching", include_unknown=True)143 assert r.models_with("function_calling", provider="gemini") == ["gemini-3.8-flash", "gemini-3.5-flash-lite"]144 assert r.providers() == ["anthropic", "gemini", "openai", "xai"]145