HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1import json2from collections import Counter34from aiatlas.connectors._identity import model_identity, split_effort_label, strip_effort5from aiatlas.connectors.benchmarks.leaderboards import (6 AiderLeaderboardConnector,7 ArtificialAnalysisConnector,8 LiveBenchLeaderboardConnector,9 SweBenchLeaderboardConnector,10 parse_rsc_models,11)12from aiatlas.sdk.facts import Target13from tests.conftest import claims_of, extract_from_fixture, fixture_path141516async def test_aider_polyglot():17 c = AiderLeaderboardConnector()18 target = Target(url=c.URL, doc_type="leaderboard", key="aider")19 facts = await extract_from_fixture(c, target, fixture_path("leaderboards", "aider-polyglot.yml"), content_type="text/plain")20 benches = {e.identifiers["registry_benchmark"] for e in facts.entities if e.entity_type == "benchmark"}21 assert benches == {"aider-polyglot", "aider-polyglot-well-formed"}22 r = next(r for r in facts.results if r.model.name == "Gemini 2.0 Pro exp-02-05" and r.metric == "pass_rate_2")23 assert r.score == 35.6 and r.unit == "%" and r.config["edit_format"] == "whole" and r.config["date"] == "2025-02-25"24 assert r.config["api_model_id"] == "gemini/gemini-2.0-pro-exp-02-05" and r.evaluated_at.year == 202525 assert r.run_group == "2025-02-25" and r.trust_level == "official-benchmark" and r.variant == "polyglot"26 assert r.model.identifiers == {"gemini_model_id": "gemini-2.0-pro-exp-02-05"} and r.model.organization.name == "Google" # pinned, genuine API id27 assert "gemini-2.0-pro-exp-02-05" in r.model.aliases and r.model.identity_confidence == "medium"28 assert not any(k in r.config for k in ("total_cost_usd", "seconds_per_case", "command", "dirname", "versions", "test_cases")) # bookkeeping dropped29 by_bench = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)30 assert by_bench["aider-polyglot-well-formed"] == by_bench["aider-polyglot"] >= 6031 wf = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "aider-polyglot-well-formed")32 assert wf.metric == "percent_cases_well_formed" and wf.variant == "well-formed"33 # effort labels are configurations of one model, never models34 gpt5 = [r for r in facts.results if r.model.name == "gpt-5" and r.metric == "pass_rate_2"]35 assert sorted(r.config["reasoning_effort"] for r in gpt5) == ["high", "low", "medium"] and len({id(r.model) for r in gpt5}) == 136 assert not any(e.name.endswith(("(high)", "(low)", "(medium)")) for e in facts.entities)37 sonnet = [r for r in facts.results if r.model.name == "claude-3-7-sonnet-20250219" and r.metric == "pass_rate_2"]38 assert {r.config.get("reasoning") for r in sonnet} == {"on", "off"} and next(r for r in sonnet if r.config.get("reasoning") == "on").config["thinking_budget"] == "32k"39 assert sonnet[0].model.identifiers == {"anthropic_model_id": "claude-3-7-sonnet-20250219"} and sonnet[0].model.organization.name == "Anthropic"40 rolling = next(r for r in facts.results if r.model.name == "DeepSeek V3 (0324)")41 assert rolling.model.identifiers == {} and "deepseek-chat" in rolling.model.aliases and rolling.model.organization.name == "DeepSeek" # rolling id → alias only42 assert not any(" + " in e.name for e in facts.entities) # architect two-model systems skipped43 assert len(facts.results) >= 120444546async def test_swebench():47 c = SweBenchLeaderboardConnector()48 facts = await extract_from_fixture(c, Target(url="https://www.swebench.com/", doc_type="leaderboard", key="swebench"), fixture_path("leaderboards", "swebench.html"))49 benches = {e.identifiers["registry_benchmark"] for e in facts.entities if e.entity_type == "benchmark"}50 assert benches == {"swe-bench-verified", "swe-bench-lite", "swe-bench-full", "swe-bench-multimodal", "swe-bench-multilingual"}51 verified = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "swe-bench-verified"]52 assert 25 <= len(verified) <= 40 and not any("&" in r.model.name for r in facts.results) # multi-model systems skipped53 assert not any(r.model.name in ("Multiple", "Undisclosed") for r in facts.results) # unnamed models skipped54 top = next(r for r in verified if r.config["system"] == "Sonar Foundation Agent")55 assert top.model.name == "Claude 4.5 Opus" and top.config["model_tag"] == "claude-opus-4-5" and top.model.organization.name == "Anthropic"56 assert top.model.identifiers == {} and "claude-opus-4-5" in top.model.aliases # free-text tag → alias, never an identifier57 assert top.score == 79.2 and top.metric == "resolved" and top.config["date"] == "2025-12-05" and top.config["open_source_system"] is False58 assert top.trust_level == "community" and top.run_group == "2025-12-05" and top.variant == "Verified"59 assert not any(k in top.config for k in ("total_cost_usd", "cost_per_instance_usd"))60 assert any(r.trust_level == "official-benchmark" and r.config["checked_by_swebench"] is True for r in facts.results)61 gemini = next(r for r in facts.results if r.model.name == "Gemini 3 Flash")62 assert gemini.model.organization.name == "Google" and gemini.model.family.name == "Gemini 3" # pattern org = the lab connector's org636465async def test_livebench_chain():66 c = LiveBenchLeaderboardConnector()67 facts = await extract_from_fixture(c, Target(url="https://livebench.ai/", doc_type="listing", key="livebench_index"), fixture_path("leaderboards", "livebench-index.html"))68 assert [t.url for t in facts.targets] == ["https://livebench.ai/static/js/main.ac6b12ef.js"]69 facts = await extract_from_fixture(c, Target(url=facts.targets[0].url, doc_type="script", key="livebench_js"), fixture_path("leaderboards", "livebench-main.js"), content_type="text/javascript")70 js_follow = facts.targets[0]71 assert js_follow.url == "https://livebench.ai/categories_2026_06_25.json" and js_follow.meta["release"] == "2026-06-25"72 assert js_follow.meta["models"]["glm-5.3"]["display_name"] == "GLM-5.3" and len(js_follow.meta["models"]) >= 27073 cats = json.loads(fixture_path("leaderboards", "livebench-categories-2026_06_25.json").read_text(encoding="utf-8"))74 facts = await extract_from_fixture(c, Target(url="https://livebench.ai/table_2026_06_25.csv", doc_type="leaderboard", key="livebench_table",75 meta={"release": "2026-06-25", "categories": cats, "models": js_follow.meta["models"]}),76 fixture_path("leaderboards", "livebench-table-2026_06_25.csv"), content_type="text/csv")77 overall = [r for r in facts.results if r.metric == "global_average"]78 assert len(overall) >= 50 and overall[0].score == 72.582 and overall[0].config["release"] == "2026-06-25"79 assert all(r.benchmark.identifiers["registry_benchmark"] == "livebench" and r.run_group == "2026-06-25" and r.trust_level == "official-benchmark" for r in overall)80 by_bench = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)81 assert by_bench == {k: len(overall) for k in ("livebench", "livebench-reasoning", "livebench-coding", "livebench-agentic-coding", "livebench-mathematics",82 "livebench-data-analysis", "livebench-language", "livebench-if")}83 reasoning = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "livebench-reasoning")84 assert reasoning.metric == "average score" and reasoning.variant == "Reasoning" and reasoning.config["variant"] == "Reasoning"85 assert not any(r.metric.startswith("category:") for r in facts.results) # no mixed metrics on one board any more86 glm = next((r for r in overall if r.model.name == "GLM-5.3"), None)87 assert glm is not None and glm.model.identifiers == {} and glm.model.organization.name == "Z.ai (Zhipu AI)"88 assert claims_of(facts, "GLM-5.3")["livebench_hf_link"] == "https://huggingface.co/zai-org/GLM-5.3"89 # effort ids fold into the base model with the setting in the configuration90 opus = next(r for r in overall if r.config["livebench_model_id"] == "claude-opus-4-6-thinking-auto-high-effort")91 assert opus.model.name == "Claude 4.6 Opus" and opus.config["reasoning_effort"] == "high" and opus.config["reasoning"] == "on" and opus.config["thinking_budget"] == "auto"92 assert "claude-opus-4-6" in opus.model.aliases and opus.model.organization.name == "Anthropic" and opus.model.identity_confidence == "medium"93 kimi = next(r for r in overall if r.config["livebench_model_id"] == "kimi-k2.6-thinking")94 assert kimi.model.name == "Kimi K2.6" and kimi.config["reasoning"] == "on"95 assert not any(e.name.lower().endswith(("effort", "xhigh", " high")) for e in facts.entities if e.entity_type == "model")969798async def test_artificial_analysis():99 c = ArtificialAnalysisConnector()100 html = fixture_path("leaderboards", "artificial-analysis-models.html").read_text(encoding="utf-8")101 models, version = parse_rsc_models(html)102 assert len(models) == 80 and version == "4.3"103 facts = await extract_from_fixture(c, Target(url=c.URL, doc_type="leaderboard", key="aa_models"), fixture_path("leaderboards", "artificial-analysis-models.html"))104 idx = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "artificial-analysis-intelligence-index"]105 assert len(idx) == 80 and all(r.config["version"] == "4.3" and r.run_group == "4.3" and r.trust_level == "independent-evaluator" for r in idx) and idx[0].unit == ""106 entities = [e for e in facts.entities if e.entity_type == "model"]107 assert len(entities) < 80 # effort variants folded → fewer models, same results108 mistral = next(r for r in idx if r.model.identifiers == {"artificial_analysis": "mistral-medium"})109 assert mistral.score == 5.49 and mistral.model.name == "Mistral Medium" and mistral.model.organization.name == "Mistral AI" and mistral.config["estimated"] is True110 assert "reasoning_effort" not in mistral.config # "-medium" is a size tier here, not an effort111 opus = next(r for r in idx if r.config["aa_slug"] == "claude-opus-5-xhigh")112 assert opus.model.name == "Claude Opus 5" and opus.config["reasoning_effort"] == "xhigh" and opus.model.organization.name == "Anthropic"113 assert "artificial_analysis" not in opus.model.identifiers and "claude-opus-5" in opus.model.aliases # AA does not list the base slug → alias only114 gpt = next(r for r in idx if r.config["aa_slug"] == "gpt-5-5")115 assert gpt.model.name == "GPT-5.5" and gpt.config["reasoning_effort"] == "xhigh" # "(xhigh)" label on a bare slug is still a setting116 kimi = [r for r in idx if r.model.name == "Kimi K2.6"]117 assert len(kimi) == 2 and len({id(r.model) for r in kimi}) == 1 and {r.config.get("reasoning") for r in kimi} == {"on", "off"}118 assert kimi[0].model.identifiers == {"artificial_analysis": "kimi-k2-6"} # base listed by AA → identifier on the base119 assert not any(e.name.lower().endswith(("(xhigh)", "(high)", "(medium)", "(low)", "(minimal)", "(non-reasoning)", "(reasoning)")) for e in entities)120 assert any(e.name == "Mistral Large 2 (Jul)" for e in entities) # snapshot month labels are not settings121 benches = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)122 assert benches["gpqa-diamond"] >= 50 and benches["humanitys-last-exam"] >= 50 and benches["terminal-bench"] >= 50 and "gpqa" not in benches123 gpqa = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "gpqa-diamond")124 assert 0 < gpqa.score <= 100 and gpqa.unit == "%" and gpqa.config["evaluator"] == "Artificial Analysis" and gpqa.variant == "Diamond"125 tau = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "tau2-bench")126 assert tau.variant == "Telecom" and tau.config["variant"] == "Telecom"127 some = next(e for e in entities if claims_of(facts, e.name).get("aa_openness"))128 assert claims_of(facts, some.name)["aa_openness"] in ("open-weights", "proprietary")129 assert not any(c.property in ("openness", "context_length", "status", "release_date") for c in facts.claims) # second-hand facts stay aa_*130 aa_only = {e.name for e in facts.entities if "artificial_analysis_creator" in e.identifiers}131 assert not aa_only & {"Upstage", "StepFun", "Thinking Machines", "Kwaipilot"} # registry organisations now, not AA-only companies132 assert all("registry_org" in e.identifiers for e in facts.entities if e.name in ("Upstage", "StepFun", "Thinking Machines"))133 qwen = next(r for r in idx if r.config["aa_slug"] == "qwen3-4b-instruct-reasoning")134 assert qwen.model.name == "Qwen3 4B" and qwen.model.organization.name == "Qwen" and qwen.config["reasoning"] == "on"135136137def test_identity_helper():138 m = model_identity("anthropic/claude-3-7-sonnet-20250219", trusted=True)139 assert m.identifiers == {"anthropic_model_id": "claude-3-7-sonnet-20250219"} and m.org_key == "anthropic" and m.pinned is True140 assert model_identity("anthropic/claude-3-7-sonnet-20250219").identifiers == {} # untrusted → aliases only141 m = model_identity("openrouter/x-ai/grok-4", trusted=True)142 assert m.identifiers == {"openrouter": "x-ai/grok-4"} and m.org_key == "xai" and "grok-4" in m.aliases143 m = model_identity("openai/Qwen/Qwen2.5-Coder-32B-Instruct", trusted=True)144 assert m.identifiers == {"hf_repo": "Qwen/Qwen2.5-Coder-32B-Instruct"} and m.org_key == "qwen" # `openai/` = OpenAI-compatible endpoint145 assert model_identity("openai/command-a-03-2025-quality").org_key == "cohere"146 assert model_identity("fireworks_ai/accounts/fireworks/models/qwq-32b", trusted=True).identifiers == {"fireworks_model_id": "fireworks/qwq-32b"}147 assert model_identity("yi-lightning").org_key is None and model_identity("r1").org_key is None148 assert strip_effort("claude-opus-4-5-20251101-thinking-64k-high-effort") == ("claude-opus-4-5-20251101", {"reasoning": "on", "thinking_budget": "64k", "reasoning_effort": "high"})149 assert strip_effort("mistral-medium") == ("mistral-medium", {}) and strip_effort("sonar-reasoning") == ("sonar-reasoning", {})150 assert strip_effort("kimi-k2-thinking") == ("kimi-k2-thinking", {}) and strip_effort("kimi-k2.6-thinking") == ("kimi-k2.6", {"reasoning": "on"})151 assert strip_effort("qwen3.7-max") == ("qwen3.7-max", {}) and strip_effort("grok-4-1-fast") == ("grok-4-1-fast", {}) # model tiers, not efforts152 assert split_effort_label("Claude Opus 5 (xhigh)") == ("Claude Opus 5", {"reasoning_effort": "xhigh"})153 assert split_effort_label("GPT-5.6 Sol xHigh Effort") == ("GPT-5.6 Sol", {"reasoning_effort": "xhigh"})154 assert split_effort_label("Kimi K2 Thinking") == ("Kimi K2 Thinking", {}) and split_effort_label("Mistral Large 2 (Jul)") == ("Mistral Large 2 (Jul)", {})155 assert split_effort_label("DeepSeek V3 (0324)") == ("DeepSeek V3 (0324)", {})156 assert split_effort_label("Mistral Small 4 (Non-reasoning)") == ("Mistral Small 4", {"reasoning": "off"})157 assert split_effort_label("gemini-2.5-pro-preview-06-05 (32k think)") == ("gemini-2.5-pro-preview-06-05", {"reasoning": "on", "thinking_budget": "32k"})158