Python 88.3%
TypeScript 7.6%
Shell 4.1%
1"""Offline tests for scripts/export_capability_graph.py against tests/shared/fixtures/*.json."""2from __future__ import annotations34import json5import sys6from pathlib import Path78ROOT = Path(__file__).resolve().parents[2]9sys.path.insert(0, str(ROOT / "scripts"))10FIX = ROOT / "tests" / "shared" / "fixtures"1112import export_capability_graph as ecg # noqa: E402131415def test_full_export(tmp_path):16 doc = ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path)17 assert {p.name for p in tmp_path.iterdir()} == {"capability-graph.json", "capability-graph.mmd", "capability-graph.dot"}18 inputs = doc["meta"]["inputs"]19 assert inputs["models"]["status"] == "loaded" and inputs["models"]["records"] == 1120 nodes = {n["id"]: n for n in doc["nodes"]}21 edges = {(e["source"], e["type"], e["target"]) for e in doc["edges"]}22 assert nodes["provider:openai"]["type"] == "provider" and nodes["provider:anthropic"]["type"] == "provider"23 assert nodes["provider:xai"]["type"] == "provider" and nodes["provider:gemini"]["type"] == "provider"24 assert doc["meta"]["providers"] == ["anthropic", "gemini", "openai", "xai"] and doc["meta"]["per_provider"]["xai"]["models"] == 2 # grok-3 is a redirect, not a model25 # xAI: retired redirect edge + alias with parenthetical note stripped26 assert ("model:xai/grok-3", "REDIRECTS_TO", "model:xai/grok-4.3") in edges and nodes["model:xai/grok-3"]["record_kind"] == "redirect"27 assert ("model:xai/grok-voice-latest", "ALIAS_OF", "model:xai/grok-voice-think-fast-2.0") in edges and "model:xai/grok-voice-latest (routes here since 2026-08-05)" not in nodes28 assert ("model:xai/grok-4.3", "SUPPORTS_UNKNOWN", "capability:web_search") in edges and ("model:xai/grok-4.3", "SUPPORTS_TOOL", "tool:xai/web_search") in edges29 assert ("tool:xai/web_search", "USABLE_ON", "endpoint:xai/POST /v1/responses") in edges and ("api_family:xai/responses", "HAS_ENDPOINT", "endpoint:xai/POST /v1/responses") in edges30 # Gemini: -latest alias record → live target, dict-shaped tools[] (support:false → no edge), templated endpoint paths31 assert ("model:gemini/gemini-flash-latest", "ALIAS_OF", "model:gemini/gemini-3.8-flash") in edges and nodes["model:gemini/gemini-flash-latest"]["record_kind"] == "alias"32 assert ("model:gemini/gemini-3.8-flash", "SUPPORTS_TOOL", "tool:gemini/google_search") in edges and ("model:gemini/gemini-3.8-flash", "SUPPORTS_TOOL", "tool:gemini/computer_use") in edges33 assert not any(e[0] == "model:gemini/gemini-3.8-flash" and e[2] == "tool:gemini/file_search" for e in edges) # support: false → no edge34 assert ("model:gemini/gemini-3.8-flash", "AVAILABLE_ON", "endpoint:gemini/POST /v1beta/models/{model}:generateContent") in edges35 assert ("tool:gemini/googleSearch", "COMPATIBLE_WITH", "model:gemini/gemini-3.8-flash") in edges36 assert ("model:gemini/gemini-3.8-flash", "SUPPORTS", "capability:structured_output") in edges # provider-native key (synonyms live in the registry, not the graph)37 assert ("provider:openai", "HAS_FAMILY", "api_family:openai/responses") in edges38 assert ("api_family:openai/responses", "HAS_ENDPOINT", "endpoint:openai/POST /v1/responses") in edges39 assert ("provider:openai", "OFFERS", "model:openai/gpt-5.4-nano") in edges40 assert ("model:openai/gpt-5.4-nano", "AVAILABLE_ON", "endpoint:openai/POST /v1/responses") in edges41 assert ("model:openai/gpt-5.4-nano", "SUPPORTS", "capability:function_calling") in edges42 assert ("model:openai/gpt-5.4-nano", "SUPPORTS_UNKNOWN", "capability:extended_thinking") in edges43 assert not any(e[0] == "model:openai/gpt-5.4-nano" and e[2] == "capability:fine_tuning" for e in edges) # false → no edge44 assert ("model:openai/gpt-5.4-nano", "SUPPORTS_TOOL", "tool:openai/web_search") in edges45 assert ("tool:openai/web_search", "USABLE_ON", "endpoint:openai/POST /v1/responses") in edges46 assert ("tool:anthropic/web_fetch_20250910", "COMPATIBLE_WITH", "model:anthropic/claude-haiku-4-5-20251001") in edges47 assert ("model:openai/gpt-5.4-nano-2026-03-17", "SNAPSHOT_OF", "model:openai/gpt-5.4-nano") in edges48 assert ("model:anthropic/claude-haiku-4-5", "ALIAS_OF", "model:anthropic/claude-haiku-4-5-20251001") in edges49 # placeholder endpoint created for a model endpoint reference absent from endpoints.json50 assert nodes["endpoint:openai/POST /v1/batches"].get("placeholder") is True51 # tool referenced by a model but without a tool record still becomes a node52 assert "tool:openai/code_interpreter" in nodes53 assert doc["meta"]["node_counts"]["provider"] == 4 and doc["meta"]["edge_counts"]["SUPPORTS"] > 10 and doc["meta"]["edge_counts"]["REDIRECTS_TO"] == 1545556def test_mermaid_and_dot_are_well_formed(tmp_path):57 ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path)58 mmd = (tmp_path / "capability-graph.mmd").read_text()59 assert mmd.startswith("graph LR\n") and "-->|SUPPORTS|" in mmd and "omitted" in mmd # alias/snapshot nodes are always omitted60 assert 'model_openai_gpt_5_4_nano(["gpt-5.4-nano"])' in mmd61 assert "claude-haiku-4-5\"" not in mmd # alias node omitted from Mermaid (kept in JSON/DOT)62 ecg.export(FIX / "models.json", FIX / "endpoints.json", FIX / "tools.json", tmp_path / "small", max_mermaid_nodes=10)63 small = (tmp_path / "small" / "capability-graph.mmd").read_text()64 assert small.count("\n ") < mmd.count("\n ") and "omitted" in small65 dot = (tmp_path / "capability-graph.dot").read_text()66 assert dot.startswith("digraph capability_graph {") and dot.rstrip().endswith("}")67 assert '"model:openai/gpt-5.4-nano" -> "capability:function_calling" [label="SUPPORTS"];' in dot68 assert dot.count("->") == len(json.loads((tmp_path / "capability-graph.json").read_text())["edges"])697071def test_partial_inputs_missing_endpoints_and_tools(tmp_path):72 doc = ecg.export(FIX / "models.json", tmp_path / "nope.json", None, tmp_path)73 assert doc["meta"]["inputs"]["endpoints"]["status"] == "missing" and doc["meta"]["inputs"]["tools"]["status"] == "missing"74 nodes = {n["id"]: n for n in doc["nodes"]}75 assert nodes["endpoint:anthropic/POST /v1/messages"]["placeholder"] is True # inferred from model.endpoints76 assert "tool:anthropic/web_fetch_20250910" in nodes # inferred from model.tools77 assert "tool:gemini/google_search" in nodes and nodes["endpoint:gemini/POST /v1beta/models/{model}:generateContent"]["placeholder"] is True78 assert doc["meta"]["node_counts"]["model"] >= 11798081def test_all_inputs_missing_or_invalid(tmp_path):82 bad = tmp_path / "bad.json"83 bad.write_text("{oops")84 doc = ecg.export(bad, tmp_path / "x.json", tmp_path / "y.json", tmp_path / "out")85 assert doc["meta"]["inputs"]["models"]["status"] == "invalid"86 assert doc["nodes"] == [] and doc["edges"] == []87 assert (tmp_path / "out" / "capability-graph.mmd").read_text() == "graph LR\n"88 assert "digraph" in (tmp_path / "out" / "capability-graph.dot").read_text()899091def test_wrapped_document_shape(tmp_path):92 models = json.loads((FIX / "models.json").read_text())93 (tmp_path / "m.json").write_text(json.dumps({"records": models}))94 doc = ecg.export(tmp_path / "m.json", None, None, tmp_path)95 assert doc["meta"]["inputs"]["models"]["records"] == 11969798def test_cli(tmp_path, capsys):99 rc = ecg.main(["--models", str(FIX / "models.json"), "--endpoints", str(FIX / "endpoints.json"), "--tools", str(FIX / "tools.json"), "--out-dir", str(tmp_path)])100 assert rc == 0 and "nodes:" in capsys.readouterr().err101