1import json2from collections import Counter34from aiatlas.connectors.benchmarks.leaderboards import (5 AiderLeaderboardConnector,6 ArtificialAnalysisConnector,7 LiveBenchLeaderboardConnector,8 SweBenchLeaderboardConnector,9 parse_rsc_models,10)11from aiatlas.sdk.facts import Target12from tests.conftest import claims_of, extract_from_fixture, fixture_path131415async def test_aider_polyglot():16 c = AiderLeaderboardConnector()17 target = Target(url=c.URL, doc_type="leaderboard", key="aider")18 facts = await extract_from_fixture(c, target, fixture_path("leaderboards", "aider-polyglot.yml"), content_type="text/plain")19 assert facts.entities[0].identifiers == {"registry_benchmark": "aider-polyglot"}20 r = next(r for r in facts.results if r.model.name == "Gemini 2.0 Pro exp-02-05" and r.metric == "pass_rate_2")21 assert r.score == 35.6 and r.unit == "%" and r.config["edit_format"] == "whole" and r.config["date"] == "2025-02-25"22 assert r.config["api_model_id"] == "gemini/gemini-2.0-pro-exp-02-05" and r.model.aliases == [] and r.evaluated_at.year == 202523 assert Counter(r.metric for r in facts.results)["percent_cases_well_formed"] == len(facts.results) // 224 assert len(facts.results) >= 100252627async def test_swebench():28 c = SweBenchLeaderboardConnector()29 facts = await extract_from_fixture(c, Target(url="https://www.swebench.com/", doc_type="leaderboard", key="swebench"), fixture_path("leaderboards", "swebench.html"))30 benches = {e.identifiers["registry_benchmark"] for e in facts.entities if e.entity_type == "benchmark"}31 assert benches == {"swe-bench-verified", "swe-bench-lite", "swe-bench-full", "swe-bench-multimodal", "swe-bench-multilingual"}32 verified = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "swe-bench-verified"]33 assert 35 <= len(verified) <= 40 and not any("&" in r.model.name for r in facts.results) # multi-model systems skipped34 top = next(r for r in verified if r.config["system"] == "Sonar Foundation Agent")35 assert top.model.name == "Claude 4.5 Opus" and top.config["model_tag"] == "claude-opus-4-5" and top.model.organization.name == "Anthropic"36 assert top.score == 79.2 and top.metric == "resolved" and top.config["date"] == "2025-12-05" and top.config["open_source_system"] is False373839async def test_livebench_chain():40 c = LiveBenchLeaderboardConnector()41 facts = await extract_from_fixture(c, Target(url="https://livebench.ai/", doc_type="listing", key="livebench_index"), fixture_path("leaderboards", "livebench-index.html"))42 assert [t.url for t in facts.targets] == ["https://livebench.ai/static/js/main.ac6b12ef.js"]43 facts = await extract_from_fixture(c, Target(url=facts.targets[0].url, doc_type="script", key="livebench_js"), fixture_path("leaderboards", "livebench-main.js"), content_type="text/javascript")44 js_follow = facts.targets[0]45 assert js_follow.url == "https://livebench.ai/categories_2026_06_25.json" and js_follow.meta["release"] == "2026-06-25"46 assert js_follow.meta["models"]["glm-5.3"]["display_name"] == "GLM-5.3" and len(js_follow.meta["models"]) >= 27047 cats = json.loads(fixture_path("leaderboards", "livebench-categories-2026_06_25.json").read_text(encoding="utf-8"))48 facts = await extract_from_fixture(c, Target(url="https://livebench.ai/table_2026_06_25.csv", doc_type="leaderboard", key="livebench_table",49 meta={"release": "2026-06-25", "categories": cats, "models": js_follow.meta["models"]}),50 fixture_path("leaderboards", "livebench-table-2026_06_25.csv"), content_type="text/csv")51 overall = [r for r in facts.results if r.metric == "global_average"]52 assert len(overall) >= 50 and overall[0].score == 72.582 and overall[0].config["release"] == "2026-06-25"53 assert any(r.metric == "category:Reasoning" for r in facts.results)54 glm = next((r for r in overall if r.model.name == "GLM-5.3"), None)55 assert glm is not None and glm.model.identifiers == {} and glm.model.organization.name == "Z.ai (Zhipu AI)"56 assert claims_of(facts, "GLM-5.3")["livebench_hf_link"] == "https://huggingface.co/zai-org/GLM-5.3"575859async def test_artificial_analysis():60 c = ArtificialAnalysisConnector()61 html = fixture_path("leaderboards", "artificial-analysis-models.html").read_text(encoding="utf-8")62 models, version = parse_rsc_models(html)63 assert len(models) == 80 and version == "4.3"64 facts = await extract_from_fixture(c, Target(url=c.URL, doc_type="leaderboard", key="aa_models"), fixture_path("leaderboards", "artificial-analysis-models.html"))65 idx = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "artificial-analysis-intelligence-index"]66 assert len(idx) == 80 and all(r.config["version"] == "4.3" for r in idx) and idx[0].unit == ""67 mistral = next(r for r in idx if r.model.identifiers == {"artificial_analysis": "mistral-medium"})68 assert mistral.score == 5.49 and mistral.model.organization.name == "Mistral AI" and mistral.config["estimated"] is True69 benches = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)70 assert benches["gpqa"] >= 50 and benches["humanitys-last-exam"] >= 50 and benches["terminal-bench"] >= 5071 gpqa = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "gpqa")72 assert 0 < gpqa.score <= 100 and gpqa.unit == "%" and gpqa.config["evaluator"] == "Artificial Analysis"73 some = next(e for e in facts.entities if e.entity_type == "model" and claims_of(facts, e.name).get("aa_openness"))74 assert claims_of(facts, some.name)["aa_openness"] in ("open-weights", "proprietary")75 assert not any(c.property in ("openness", "context_length", "status", "release_date") for c in facts.claims) # second-hand facts stay aa_*76