import json
from collections import Counter

from aiatlas.connectors._identity import model_identity, split_effort_label, strip_effort
from aiatlas.connectors.benchmarks.leaderboards import (
    AiderLeaderboardConnector,
    ArtificialAnalysisConnector,
    LiveBenchLeaderboardConnector,
    SweBenchLeaderboardConnector,
    parse_rsc_models,
)
from aiatlas.sdk.facts import Target
from tests.conftest import claims_of, extract_from_fixture, fixture_path


async def test_aider_polyglot():
    c = AiderLeaderboardConnector()
    target = Target(url=c.URL, doc_type="leaderboard", key="aider")
    facts = await extract_from_fixture(c, target, fixture_path("leaderboards", "aider-polyglot.yml"), content_type="text/plain")
    benches = {e.identifiers["registry_benchmark"] for e in facts.entities if e.entity_type == "benchmark"}
    assert benches == {"aider-polyglot", "aider-polyglot-well-formed"}
    r = next(r for r in facts.results if r.model.name == "Gemini 2.0 Pro exp-02-05" and r.metric == "pass_rate_2")
    assert r.score == 35.6 and r.unit == "%" and r.config["edit_format"] == "whole" and r.config["date"] == "2025-02-25"
    assert r.config["api_model_id"] == "gemini/gemini-2.0-pro-exp-02-05" and r.evaluated_at.year == 2025
    assert r.run_group == "2025-02-25" and r.trust_level == "official-benchmark" and r.variant == "polyglot"
    assert r.model.identifiers == {"gemini_model_id": "gemini-2.0-pro-exp-02-05"} and r.model.organization.name == "Google"   # pinned, genuine API id
    assert "gemini-2.0-pro-exp-02-05" in r.model.aliases and r.model.identity_confidence == "medium"
    assert not any(k in r.config for k in ("total_cost_usd", "seconds_per_case", "command", "dirname", "versions", "test_cases"))   # bookkeeping dropped
    by_bench = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)
    assert by_bench["aider-polyglot-well-formed"] == by_bench["aider-polyglot"] >= 60
    wf = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "aider-polyglot-well-formed")
    assert wf.metric == "percent_cases_well_formed" and wf.variant == "well-formed"
    # effort labels are configurations of one model, never models
    gpt5 = [r for r in facts.results if r.model.name == "gpt-5" and r.metric == "pass_rate_2"]
    assert sorted(r.config["reasoning_effort"] for r in gpt5) == ["high", "low", "medium"] and len({id(r.model) for r in gpt5}) == 1
    assert not any(e.name.endswith(("(high)", "(low)", "(medium)")) for e in facts.entities)
    sonnet = [r for r in facts.results if r.model.name == "claude-3-7-sonnet-20250219" and r.metric == "pass_rate_2"]
    assert {r.config.get("reasoning") for r in sonnet} == {"on", "off"} and next(r for r in sonnet if r.config.get("reasoning") == "on").config["thinking_budget"] == "32k"
    assert sonnet[0].model.identifiers == {"anthropic_model_id": "claude-3-7-sonnet-20250219"} and sonnet[0].model.organization.name == "Anthropic"
    rolling = next(r for r in facts.results if r.model.name == "DeepSeek V3 (0324)")
    assert rolling.model.identifiers == {} and "deepseek-chat" in rolling.model.aliases and rolling.model.organization.name == "DeepSeek"   # rolling id → alias only
    assert not any(" + " in e.name for e in facts.entities)                                          # architect two-model systems skipped
    assert len(facts.results) >= 120


async def test_swebench():
    c = SweBenchLeaderboardConnector()
    facts = await extract_from_fixture(c, Target(url="https://www.swebench.com/", doc_type="leaderboard", key="swebench"), fixture_path("leaderboards", "swebench.html"))
    benches = {e.identifiers["registry_benchmark"] for e in facts.entities if e.entity_type == "benchmark"}
    assert benches == {"swe-bench-verified", "swe-bench-lite", "swe-bench-full", "swe-bench-multimodal", "swe-bench-multilingual"}
    verified = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "swe-bench-verified"]
    assert 25 <= len(verified) <= 40 and not any("&" in r.model.name for r in facts.results)      # multi-model systems skipped
    assert not any(r.model.name in ("Multiple", "Undisclosed") for r in facts.results)              # unnamed models skipped
    top = next(r for r in verified if r.config["system"] == "Sonar Foundation Agent")
    assert top.model.name == "Claude 4.5 Opus" and top.config["model_tag"] == "claude-opus-4-5" and top.model.organization.name == "Anthropic"
    assert top.model.identifiers == {} and "claude-opus-4-5" in top.model.aliases                    # free-text tag → alias, never an identifier
    assert top.score == 79.2 and top.metric == "resolved" and top.config["date"] == "2025-12-05" and top.config["open_source_system"] is False
    assert top.trust_level == "community" and top.run_group == "2025-12-05" and top.variant == "Verified"
    assert not any(k in top.config for k in ("total_cost_usd", "cost_per_instance_usd"))
    assert any(r.trust_level == "official-benchmark" and r.config["checked_by_swebench"] is True for r in facts.results)
    gemini = next(r for r in facts.results if r.model.name == "Gemini 3 Flash")
    assert gemini.model.organization.name == "Google" and gemini.model.family.name == "Gemini 3"     # pattern org = the lab connector's org


async def test_livebench_chain():
    c = LiveBenchLeaderboardConnector()
    facts = await extract_from_fixture(c, Target(url="https://livebench.ai/", doc_type="listing", key="livebench_index"), fixture_path("leaderboards", "livebench-index.html"))
    assert [t.url for t in facts.targets] == ["https://livebench.ai/static/js/main.ac6b12ef.js"]
    facts = await extract_from_fixture(c, Target(url=facts.targets[0].url, doc_type="script", key="livebench_js"), fixture_path("leaderboards", "livebench-main.js"), content_type="text/javascript")
    js_follow = facts.targets[0]
    assert js_follow.url == "https://livebench.ai/categories_2026_06_25.json" and js_follow.meta["release"] == "2026-06-25"
    assert js_follow.meta["models"]["glm-5.3"]["display_name"] == "GLM-5.3" and len(js_follow.meta["models"]) >= 270
    cats = json.loads(fixture_path("leaderboards", "livebench-categories-2026_06_25.json").read_text(encoding="utf-8"))
    facts = await extract_from_fixture(c, Target(url="https://livebench.ai/table_2026_06_25.csv", doc_type="leaderboard", key="livebench_table",
                                                 meta={"release": "2026-06-25", "categories": cats, "models": js_follow.meta["models"]}),
                                       fixture_path("leaderboards", "livebench-table-2026_06_25.csv"), content_type="text/csv")
    overall = [r for r in facts.results if r.metric == "global_average"]
    assert len(overall) >= 50 and overall[0].score == 72.582 and overall[0].config["release"] == "2026-06-25"
    assert all(r.benchmark.identifiers["registry_benchmark"] == "livebench" and r.run_group == "2026-06-25" and r.trust_level == "official-benchmark" for r in overall)
    by_bench = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)
    assert by_bench == {k: len(overall) for k in ("livebench", "livebench-reasoning", "livebench-coding", "livebench-agentic-coding", "livebench-mathematics",
                                                   "livebench-data-analysis", "livebench-language", "livebench-if")}
    reasoning = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "livebench-reasoning")
    assert reasoning.metric == "average score" and reasoning.variant == "Reasoning" and reasoning.config["variant"] == "Reasoning"
    assert not any(r.metric.startswith("category:") for r in facts.results)                          # no mixed metrics on one board any more
    glm = next((r for r in overall if r.model.name == "GLM-5.3"), None)
    assert glm is not None and glm.model.identifiers == {} and glm.model.organization.name == "Z.ai (Zhipu AI)"
    assert claims_of(facts, "GLM-5.3")["livebench_hf_link"] == "https://huggingface.co/zai-org/GLM-5.3"
    # effort ids fold into the base model with the setting in the configuration
    opus = next(r for r in overall if r.config["livebench_model_id"] == "claude-opus-4-6-thinking-auto-high-effort")
    assert opus.model.name == "Claude 4.6 Opus" and opus.config["reasoning_effort"] == "high" and opus.config["reasoning"] == "on" and opus.config["thinking_budget"] == "auto"
    assert "claude-opus-4-6" in opus.model.aliases and opus.model.organization.name == "Anthropic" and opus.model.identity_confidence == "medium"
    kimi = next(r for r in overall if r.config["livebench_model_id"] == "kimi-k2.6-thinking")
    assert kimi.model.name == "Kimi K2.6" and kimi.config["reasoning"] == "on"
    assert not any(e.name.lower().endswith(("effort", "xhigh", " high")) for e in facts.entities if e.entity_type == "model")


async def test_artificial_analysis():
    c = ArtificialAnalysisConnector()
    html = fixture_path("leaderboards", "artificial-analysis-models.html").read_text(encoding="utf-8")
    models, version = parse_rsc_models(html)
    assert len(models) == 80 and version == "4.3"
    facts = await extract_from_fixture(c, Target(url=c.URL, doc_type="leaderboard", key="aa_models"), fixture_path("leaderboards", "artificial-analysis-models.html"))
    idx = [r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "artificial-analysis-intelligence-index"]
    assert len(idx) == 80 and all(r.config["version"] == "4.3" and r.run_group == "4.3" and r.trust_level == "independent-evaluator" for r in idx) and idx[0].unit == ""
    entities = [e for e in facts.entities if e.entity_type == "model"]
    assert len(entities) < 80                                                                         # effort variants folded → fewer models, same results
    mistral = next(r for r in idx if r.model.identifiers == {"artificial_analysis": "mistral-medium"})
    assert mistral.score == 5.49 and mistral.model.name == "Mistral Medium" and mistral.model.organization.name == "Mistral AI" and mistral.config["estimated"] is True
    assert "reasoning_effort" not in mistral.config                                                   # "-medium" is a size tier here, not an effort
    opus = next(r for r in idx if r.config["aa_slug"] == "claude-opus-5-xhigh")
    assert opus.model.name == "Claude Opus 5" and opus.config["reasoning_effort"] == "xhigh" and opus.model.organization.name == "Anthropic"
    assert "artificial_analysis" not in opus.model.identifiers and "claude-opus-5" in opus.model.aliases    # AA does not list the base slug → alias only
    gpt = next(r for r in idx if r.config["aa_slug"] == "gpt-5-5")
    assert gpt.model.name == "GPT-5.5" and gpt.config["reasoning_effort"] == "xhigh"                # "(xhigh)" label on a bare slug is still a setting
    kimi = [r for r in idx if r.model.name == "Kimi K2.6"]
    assert len(kimi) == 2 and len({id(r.model) for r in kimi}) == 1 and {r.config.get("reasoning") for r in kimi} == {"on", "off"}
    assert kimi[0].model.identifiers == {"artificial_analysis": "kimi-k2-6"}                        # base listed by AA → identifier on the base
    assert not any(e.name.lower().endswith(("(xhigh)", "(high)", "(medium)", "(low)", "(minimal)", "(non-reasoning)", "(reasoning)")) for e in entities)
    assert any(e.name == "Mistral Large 2 (Jul)" for e in entities)                                 # snapshot month labels are not settings
    benches = Counter(r.benchmark.identifiers["registry_benchmark"] for r in facts.results)
    assert benches["gpqa-diamond"] >= 50 and benches["humanitys-last-exam"] >= 50 and benches["terminal-bench"] >= 50 and "gpqa" not in benches
    gpqa = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "gpqa-diamond")
    assert 0 < gpqa.score <= 100 and gpqa.unit == "%" and gpqa.config["evaluator"] == "Artificial Analysis" and gpqa.variant == "Diamond"
    tau = next(r for r in facts.results if r.benchmark.identifiers["registry_benchmark"] == "tau2-bench")
    assert tau.variant == "Telecom" and tau.config["variant"] == "Telecom"
    some = next(e for e in entities if claims_of(facts, e.name).get("aa_openness"))
    assert claims_of(facts, some.name)["aa_openness"] in ("open-weights", "proprietary")
    assert not any(c.property in ("openness", "context_length", "status", "release_date") for c in facts.claims)   # second-hand facts stay aa_*
    aa_only = {e.name for e in facts.entities if "artificial_analysis_creator" in e.identifiers}
    assert not aa_only & {"Upstage", "StepFun", "Thinking Machines", "Kwaipilot"}                    # registry organisations now, not AA-only companies
    assert all("registry_org" in e.identifiers for e in facts.entities if e.name in ("Upstage", "StepFun", "Thinking Machines"))
    qwen = next(r for r in idx if r.config["aa_slug"] == "qwen3-4b-instruct-reasoning")
    assert qwen.model.name == "Qwen3 4B" and qwen.model.organization.name == "Qwen" and qwen.config["reasoning"] == "on"


def test_identity_helper():
    m = model_identity("anthropic/claude-3-7-sonnet-20250219", trusted=True)
    assert m.identifiers == {"anthropic_model_id": "claude-3-7-sonnet-20250219"} and m.org_key == "anthropic" and m.pinned is True
    assert model_identity("anthropic/claude-3-7-sonnet-20250219").identifiers == {}                 # untrusted → aliases only
    m = model_identity("openrouter/x-ai/grok-4", trusted=True)
    assert m.identifiers == {"openrouter": "x-ai/grok-4"} and m.org_key == "xai" and "grok-4" in m.aliases
    m = model_identity("openai/Qwen/Qwen2.5-Coder-32B-Instruct", trusted=True)
    assert m.identifiers == {"hf_repo": "Qwen/Qwen2.5-Coder-32B-Instruct"} and m.org_key == "qwen"   # `openai/` = OpenAI-compatible endpoint
    assert model_identity("openai/command-a-03-2025-quality").org_key == "cohere"
    assert model_identity("fireworks_ai/accounts/fireworks/models/qwq-32b", trusted=True).identifiers == {"fireworks_model_id": "fireworks/qwq-32b"}
    assert model_identity("yi-lightning").org_key is None and model_identity("r1").org_key is None
    assert strip_effort("claude-opus-4-5-20251101-thinking-64k-high-effort") == ("claude-opus-4-5-20251101", {"reasoning": "on", "thinking_budget": "64k", "reasoning_effort": "high"})
    assert strip_effort("mistral-medium") == ("mistral-medium", {}) and strip_effort("sonar-reasoning") == ("sonar-reasoning", {})
    assert strip_effort("kimi-k2-thinking") == ("kimi-k2-thinking", {}) and strip_effort("kimi-k2.6-thinking") == ("kimi-k2.6", {"reasoning": "on"})
    assert strip_effort("qwen3.7-max") == ("qwen3.7-max", {}) and strip_effort("grok-4-1-fast") == ("grok-4-1-fast", {})   # model tiers, not efforts
    assert split_effort_label("Claude Opus 5 (xhigh)") == ("Claude Opus 5", {"reasoning_effort": "xhigh"})
    assert split_effort_label("GPT-5.6 Sol xHigh Effort") == ("GPT-5.6 Sol", {"reasoning_effort": "xhigh"})
    assert split_effort_label("Kimi K2 Thinking") == ("Kimi K2 Thinking", {}) and split_effort_label("Mistral Large 2 (Jul)") == ("Mistral Large 2 (Jul)", {})
    assert split_effort_label("DeepSeek V3 (0324)") == ("DeepSeek V3 (0324)", {})
    assert split_effort_label("Mistral Small 4 (Non-reasoning)") == ("Mistral Small 4", {"reasoning": "off"})
    assert split_effort_label("gemini-2.5-pro-preview-06-05 (32k think)") == ("gemini-2.5-pro-preview-06-05", {"reasoning": "on", "thinking_budget": "32k"})