"""Offline tests for scripts/export_capability_graph.py against tests/shared/fixtures/*.json.""" from __future__ import annotations import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts")) FIX = ROOT / "tests" / "shared" / "fixtures" import export_capability_graph as ecg # noqa: E402 def test_full_export(tmp_path): doc = ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path) assert {p.name for p in tmp_path.iterdir()} == {"capability-graph.json", "capability-graph.mmd", "capability-graph.dot"} inputs = doc["meta"]["inputs"] assert inputs["models"]["status"] == "loaded" and inputs["models"]["records"] == 11 nodes = {n["id"]: n for n in doc["nodes"]} edges = {(e["source"], e["type"], e["target"]) for e in doc["edges"]} assert nodes["provider:openai"]["type"] == "provider" and nodes["provider:anthropic"]["type"] == "provider" assert nodes["provider:xai"]["type"] == "provider" and nodes["provider:gemini"]["type"] == "provider" assert doc["meta"]["providers"] == ["anthropic", "gemini", "openai", "xai"] and doc["meta"]["per_provider"]["xai"]["models"] == 2 # grok-3 is a redirect, not a model # xAI: retired redirect edge + alias with parenthetical note stripped assert ("model:xai/grok-3", "REDIRECTS_TO", "model:xai/grok-4.3") in edges and nodes["model:xai/grok-3"]["record_kind"] == "redirect" assert ("model:xai/grok-voice-latest", "ALIAS_OF", "model:xai/grok-voice-think-fast-2.0") in edges and "model:xai/grok-voice-latest (routes here since 2026-08-05)" not in nodes assert ("model:xai/grok-4.3", "SUPPORTS_UNKNOWN", "capability:web_search") in edges and ("model:xai/grok-4.3", "SUPPORTS_TOOL", "tool:xai/web_search") in edges assert ("tool:xai/web_search", "USABLE_ON", "endpoint:xai/POST /v1/responses") in edges and ("api_family:xai/responses", "HAS_ENDPOINT", "endpoint:xai/POST /v1/responses") in edges # Gemini: -latest alias record → live target, dict-shaped tools[] (support:false → no edge), templated endpoint paths assert ("model:gemini/gemini-flash-latest", "ALIAS_OF", "model:gemini/gemini-3.8-flash") in edges and nodes["model:gemini/gemini-flash-latest"]["record_kind"] == "alias" assert ("model:gemini/gemini-3.8-flash", "SUPPORTS_TOOL", "tool:gemini/google_search") in edges and ("model:gemini/gemini-3.8-flash", "SUPPORTS_TOOL", "tool:gemini/computer_use") in edges assert not any(e[0] == "model:gemini/gemini-3.8-flash" and e[2] == "tool:gemini/file_search" for e in edges) # support: false → no edge assert ("model:gemini/gemini-3.8-flash", "AVAILABLE_ON", "endpoint:gemini/POST /v1beta/models/{model}:generateContent") in edges assert ("tool:gemini/googleSearch", "COMPATIBLE_WITH", "model:gemini/gemini-3.8-flash") in edges assert ("model:gemini/gemini-3.8-flash", "SUPPORTS", "capability:structured_output") in edges # provider-native key (synonyms live in the registry, not the graph) assert ("provider:openai", "HAS_FAMILY", "api_family:openai/responses") in edges assert ("api_family:openai/responses", "HAS_ENDPOINT", "endpoint:openai/POST /v1/responses") in edges assert ("provider:openai", "OFFERS", "model:openai/gpt-5.4-nano") in edges assert ("model:openai/gpt-5.4-nano", "AVAILABLE_ON", "endpoint:openai/POST /v1/responses") in edges assert ("model:openai/gpt-5.4-nano", "SUPPORTS", "capability:function_calling") in edges assert ("model:openai/gpt-5.4-nano", "SUPPORTS_UNKNOWN", "capability:extended_thinking") in edges assert not any(e[0] == "model:openai/gpt-5.4-nano" and e[2] == "capability:fine_tuning" for e in edges) # false → no edge assert ("model:openai/gpt-5.4-nano", "SUPPORTS_TOOL", "tool:openai/web_search") in edges assert ("tool:openai/web_search", "USABLE_ON", "endpoint:openai/POST /v1/responses") in edges assert ("tool:anthropic/web_fetch_20250910", "COMPATIBLE_WITH", "model:anthropic/claude-haiku-4-5-20251001") in edges assert ("model:openai/gpt-5.4-nano-2026-03-17", "SNAPSHOT_OF", "model:openai/gpt-5.4-nano") in edges assert ("model:anthropic/claude-haiku-4-5", "ALIAS_OF", "model:anthropic/claude-haiku-4-5-20251001") in edges # placeholder endpoint created for a model endpoint reference absent from endpoints.json assert nodes["endpoint:openai/POST /v1/batches"].get("placeholder") is True # tool referenced by a model but without a tool record still becomes a node assert "tool:openai/code_interpreter" in nodes assert doc["meta"]["node_counts"]["provider"] == 4 and doc["meta"]["edge_counts"]["SUPPORTS"] > 10 and doc["meta"]["edge_counts"]["REDIRECTS_TO"] == 1 def test_mermaid_and_dot_are_well_formed(tmp_path): ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path) mmd = (tmp_path / "capability-graph.mmd").read_text() assert mmd.startswith("graph LR\n") and "-->|SUPPORTS|" in mmd and "omitted" in mmd # alias/snapshot nodes are always omitted assert 'model_openai_gpt_5_4_nano(["gpt-5.4-nano"])' in mmd assert "claude-haiku-4-5\"" not in mmd # alias node omitted from Mermaid (kept in JSON/DOT) ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path / "small", max_mermaid_nodes=10) small = (tmp_path / "small" / "capability-graph.mmd").read_text() assert small.count("\n ") < mmd.count("\n ") and "omitted" in small dot = (tmp_path / "capability-graph.dot").read_text() assert dot.startswith("digraph capability_graph {") and dot.rstrip().endswith("}") assert '"model:openai/gpt-5.4-nano" -> "capability:function_calling" [label="SUPPORTS"];' in dot assert dot.count("->") == len(json.loads((tmp_path / "capability-graph.json").read_text())["edges"]) def test_partial_inputs_missing_endpoints_and_tools(tmp_path): doc = ecg.export(FIX / "models.json", tmp_path / "nope.json", None, tmp_path) assert doc["meta"]["inputs"]["endpoints"]["status"] == "missing" and doc["meta"]["inputs"]["tools"]["status"] == "missing" nodes = {n["id"]: n for n in doc["nodes"]} assert nodes["endpoint:anthropic/POST /v1/messages"]["placeholder"] is True # inferred from model.endpoints assert "tool:anthropic/web_fetch_20250910" in nodes # inferred from model.tools assert "tool:gemini/google_search" in nodes and nodes["endpoint:gemini/POST /v1beta/models/{model}:generateContent"]["placeholder"] is True assert doc["meta"]["node_counts"]["model"] >= 11 def test_all_inputs_missing_or_invalid(tmp_path): bad = tmp_path / "bad.json" bad.write_text("{oops") doc = ecg.export(bad, tmp_path / "x.json", tmp_path / "y.json", tmp_path / "out") assert doc["meta"]["inputs"]["models"]["status"] == "invalid" assert doc["nodes"] == [] and doc["edges"] == [] assert (tmp_path / "out" / "capability-graph.mmd").read_text() == "graph LR\n" assert "digraph" in (tmp_path / "out" / "capability-graph.dot").read_text() def test_wrapped_document_shape(tmp_path): models = json.loads((FIX / "models.json").read_text()) (tmp_path / "m.json").write_text(json.dumps({"records": models})) doc = ecg.export(tmp_path / "m.json", None, None, tmp_path) assert doc["meta"]["inputs"]["models"]["records"] == 11 def test_cli(tmp_path, capsys): rc = ecg.main(["--models", str(FIX / "models.json"), "--endpoints", str(FIX / "endpoints.json"), "--tools", str(FIX / "tools.json"), "--out-dir", str(tmp_path)]) assert rc == 0 and "nodes:" in capsys.readouterr().err