"""Offline tests for examples/shared/feature-detection/supports.py using tests/shared/fixtures/{models,tools}.json.""" from __future__ import annotations import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "examples" / "shared" / "feature-detection")) FIX = ROOT / "tests" / "shared" / "fixtures" from supports import Registry, UNKNOWN # noqa: E402 def reg() -> Registry: return Registry.load(FIX / "models.json", FIX / "tools.json") def test_tri_state_values(): r = reg() assert r.supports("gpt-5.4-nano", "function_calling") is True assert r.supports("gpt-5.4-nano", "fine_tuning") is False assert r.supports("gpt-5.4-nano", "extended_thinking") == UNKNOWN # explicit "unknown" assert r.supports("gpt-5.4-nano", "never_heard_of_it") == UNKNOWN # missing key assert r.supports("ada", "function_calling") == UNKNOWN # capabilities object without booleans assert r.supports("does-not-exist", "streaming") == UNKNOWN # unknown model def test_alias_and_snapshot_resolution(): r = reg() assert r.canonical_id("claude-haiku-4-5") == "claude-haiku-4-5-20251001" assert r.supports("claude-haiku-4-5", "extended_thinking") is True assert r.canonical_id("gpt-5.4-nano-2026-03-17") == "gpt-5.4-nano" # snapshot listed + snapshot-kind record defers assert r.canonical_id("gpt-5.4-nano-latest") == "gpt-5.4-nano" # alias assert r.canonical_id("GPT-5.4-NANO") == "gpt-5.4-nano" # case-insensitive assert r.canonical_id("openai/gpt-5.4-nano") == "gpt-5.4-nano" # provider prefix assert r.canonical_id("claude-haiku-4-5-20251001-20260101") == "claude-haiku-4-5-20251001" # unlisted date suffix assert r.canonical_id("nope") is None def test_models_with(): r = reg() both = r.models_with("function_calling", "structured_outputs") # deprecated opus excluded, snapshot excluded, xAI retired_redirect + Gemini alias-kind records excluded, voice model (structured_outputs false) excluded assert both == ["gpt-5.4-nano", "claude-haiku-4-5-20251001", "grok-4.3", "gemini-3.8-flash", "gemini-3.5-flash-lite"] assert r.models_with("function_calling", provider="anthropic") == ["claude-haiku-4-5-20251001"] assert r.models_with("extended_thinking", provider="anthropic") == ["claude-haiku-4-5-20251001"] assert r.models_with("extended_thinking", include_unknown=True, provider="openai") == ["gpt-5.4-nano"] assert "claude-opus-4-1-20250805" in r.models_with("function_calling", statuses_excluded=()) assert "grok-3" not in r.models_with("function_calling", statuses_excluded=()) # pointer records never listed def test_tools_for_and_supports_tool(): r = reg() tools = r.tools_for("claude-haiku-4-5") assert "web_fetch_20250910" in tools and "mcp_toolset" in tools assert r.supports_tool("claude-haiku-4-5", "web_fetch") is True # name matches versioned type assert r.supports_tool("claude-haiku-4-5", "web_search_20250305") is True assert r.supports_tool("gpt-5.4-nano", "web_search") is True assert r.supports_tool("gpt-5.4-nano", "computer_use_preview") is False # explicit tools[] list, not present assert r.supports_tool("ada", "web_search") == UNKNOWN # empty tools[], no tool record assert r.supports("gpt-5.4-nano", "tool_computer_use") is False assert r.supports("claude-haiku-4-5", "tool_code_execution") == UNKNOWN # explicit "unknown" flag wins over inference assert r.supports("claude-opus-4-1", "tool_web_search") is True # inferred from tools[] assert r.supports_tool("claude-opus-4-1", "web_fetch") is False # tool record lists compatible models, not this one def test_missing_or_malformed_inputs_are_graceful(tmp_path): r = Registry.load(tmp_path / "missing.json", None) assert r.models == [] and r.supports("x", "y") == UNKNOWN and r.tools_for("x") == [] bad = tmp_path / "bad.json" bad.write_text("{not json") assert Registry.load(bad, bad).models == [] def test_alternative_document_shapes(): recs = json.loads((FIX / "models.json").read_text()) wrapped = Registry.from_documents({"records": recs}, {"tools": json.loads((FIX / "tools.json").read_text())}) assert wrapped.supports("gpt-5.4-nano", "streaming") is True and len(wrapped.tools) == 9 keyed = Registry.from_documents({m["id"]: {k: v for k, v in m.items() if k != "id"} for m in recs}) assert keyed.supports("claude-haiku-4-5", "prompt_caching") is True def test_compare(): r = reg() c = r.compare("gpt-5.4-nano", "claude-haiku-4-5") assert c["tool_computer_use"] == (False, True) and c["extended_thinking"] == (UNKNOWN, True) # ---------------- xAI records (aliases, retired redirects, parenthetical alias notes) ---------------- def test_xai_alias_and_retired_redirect_resolution(): r = reg() assert r.canonical_id("grok-4.3-latest") == "grok-4.3" and r.canonical_id("xai/grok-4.3") == "grok-4.3" assert r.canonical_id("grok-3") == "grok-4.3" # kind: retired_redirect → target parsed from verification.request_note assert r.supports("grok-3", "structured_outputs") is True # queries on the retired id answer for the live target assert r.canonical_id("grok-voice-latest") == "grok-voice-think-fast-2.0" # "grok-voice-latest (routes here since …)" note stripped assert Registry.redirect_target({"kind": "retired_redirect", "verification": {"request_note": "GET /v1/models/x -> 200 with id 'grok-4.3'"}}) == "grok-4.3" assert Registry.redirect_target({"kind": "model"}) is None def test_xai_capabilities_unknown_server_tools_and_tool_inference(): r = reg() assert r.supports("grok-4.3", "function_calling") is True and r.supports("grok-4.3", "structured_outputs") is True assert r.supports("grok-4.3", "web_search") == UNKNOWN # explicit "unknown" flag wins over the tools[] entry assert r.supports_tool("grok-4.3", "web_search") is True # but the tool list itself says the type string is accepted assert r.supports("grok-4.3", "prompt_caching") is True # synonym prompt_caching_automatic assert r.supports("grok-4.3", "reasoning") is True and r.supports("grok-4.3", "fine_tuning") is False assert r.supports_tool("grok-4.3", "image_generation") is False # explicit list, absent assert "mcp" in r.tools_for("grok-4.3") and r.tool_record("xai", "web_search")["category"] == "server" # ---------------- Gemini records (models/ prefix, -latest dict aliases, dict-shaped tools[], synonyms) ---------------- def test_gemini_prefix_and_latest_alias_resolution(): r = reg() assert r.canonical_id("models/gemini-3.8-flash") == "gemini-3.8-flash" assert r.canonical_id("gemini/models/gemini-3.8-flash") == "gemini-3.8-flash" assert r.canonical_id("gemini-flash-latest") == "gemini-3.8-flash" # alias record → resolves_to_live (live 2026-09-19) assert r.canonical_id("models/gemini-flash-latest") == "gemini-3.8-flash" assert r.supports("gemini-flash-latest", "function_calling") is True # answered by the target record, not the alias stub def test_gemini_dict_tools_and_camel_snake_matching(): r = reg() assert r.supports_tool("gemini-3.8-flash", "google_search") is True and r.supports_tool("gemini-3.8-flash", "googleSearch") is True assert r.supports_tool("gemini-3.8-flash", "file_search") is False # support: false in the dict entry assert r.supports_tool("gemini-3.8-flash", "computer_use") is True # "Supported (Preview)" string → True assert r.supports_tool("gemini-3.8-flash", "google_maps") is False # explicit list, absent tools = r.tools_for("gemini-3.8-flash") assert "google_search" in tools and "file_search" not in tools and "urlContext" not in tools and "url_context" in tools # tool record deduped by normalised key assert r.supports("gemini-3.8-flash", "tool_code_execution") is True and r.supports("gemini-3.8-flash", "computer_use") == UNKNOWN # explicit flag wins def test_cross_provider_synonyms_in_models_with(): r = reg() assert r.supports("gemini-3.8-flash", "structured_outputs") is True # ← structured_output assert r.supports("gemini-3.8-flash", "web_search") is True # ← google_search_grounding assert r.supports("gemini-3.8-flash", "prompt_caching") is True # ← context_caching_implicit assert r.models_with("web_search", "structured_outputs", "prompt_caching") == ["gpt-5.4-nano", "claude-haiku-4-5-20251001", "gemini-3.8-flash", "gemini-3.5-flash-lite"] # grok-4.3 is excluded because its web_search flag is an explicit "unknown" (include_unknown lets it through) assert "grok-4.3" in r.models_with("web_search", "structured_outputs", "prompt_caching", include_unknown=True) assert r.models_with("function_calling", provider="gemini") == ["gemini-3.8-flash", "gemini-3.5-flash-lite"] assert r.providers() == ["anthropic", "gemini", "openai", "xai"]