HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Ontology — pure functions, no database."""2from __future__ import annotations34import pytest56from aiatlas.ontology import benchmarks as b7from aiatlas.ontology.licenses import LICENSES, license_info, normalize_license8from aiatlas.ontology.models import analyze_model_name, base_name, effort_config, family_hint, family_release_hint, is_official_org, variant_key9from aiatlas.ontology.openness import derive_openness, normalize_openness, openness_dimensions10from aiatlas.ontology.taxonomy import normalize_hardware_kind, normalize_modalities, normalize_org_kind, normalize_property, normalize_status111213# ---------------------------------------------------------------------------------------------- licences14@pytest.mark.parametrize("raw", ["Apache 2.0", "apache-2.0", "Apache-2.0", "Apache License 2.0", "apache license, version 2.0", "APACHE2", "License: apache-2.0"])15def test_apache_variants_one_key(raw: str) -> None:16 assert normalize_license(raw) == "Apache-2.0"171819@pytest.mark.parametrize("raw,key", [20 ("mit", "MIT"), ("Modified MIT", "MIT-Modified"), ("CC BY-NC 4.0", "CC-BY-NC-4.0"), ("cc-by-nc-4.0", "CC-BY-NC-4.0"), ("cc-by-4.0", "CC-BY-4.0"),21 ("llama3", "Llama-3-Community"), ("llama3.1", "Llama-3.1-Community"), ("llama3.2", "Llama-3.2-Community"), ("Llama 4 Community License", "Llama-4-Community"),22 ("gemma", "Gemma-Terms"), ("openrail++", "OpenRAIL++-M"), ("bigscience-bloom-rail-1.0", "BigScience-BLOOM-RAIL-1.0"), ("other", "Other"),23 ("apple-amlr", "Apple-AMLR"), ("bsd-3-clause", "BSD-3-Clause"), ("proprietary", "Proprietary"), ("GPLv3", "GPL-3.0"),24])25def test_license_table(raw: str, key: str) -> None:26 assert normalize_license(raw) == key27 assert key in LICENSES282930def test_unknown_license_is_none_not_guessed() -> None:31 assert normalize_license("weird-license-xyz") is None32 assert normalize_license("") is None and normalize_license(None) is None33 assert license_info("weird") is None343536# ---------------------------------------------------------------------------------------------- openness37def test_openness_derivation() -> None:38 llama = openness_dimensions(weights_available=True, license_key="Llama-3.1-Community")39 assert derive_openness(llama, license_key="Llama-3.1-Community") == "restricted-weights"40 apache = openness_dimensions(weights_available=True, license_key="Apache-2.0")41 assert derive_openness(apache, license_key="Apache-2.0") == "open-weights"42 assert derive_openness({**apache, "source_code_available": True}, license_key="Apache-2.0") == "open-source"43 prop = openness_dimensions(weights_available=None, license_key="Proprietary")44 assert prop["weights_available"] is False and derive_openness(prop, license_key="Proprietary") == "proprietary"45 assert derive_openness(openness_dimensions(weights_available=None), license_key=None) == "unknown"46 nc = openness_dimensions(weights_available=True, license_key="CC-BY-NC-4.0")47 assert nc["commercial_use_allowed"] is False and derive_openness(nc, license_key="CC-BY-NC-4.0") == "restricted-weights"484950@pytest.mark.parametrize("raw,canon", [("open-weights", "open-weights"), ("restricted", "restricted-weights"), ("gated", "restricted-weights"), ("closed", "proprietary"),51 ("Open Source", "open-source"), ("", "unknown"), ("banana", None)])52def test_normalize_openness(raw: str, canon: str | None) -> None:53 assert normalize_openness(raw) == canon545556# ---------------------------------------------------------------------------------------------- taxonomy57def test_modalities_and_status_and_kinds() -> None:58 assert normalize_modalities(["Text", "text", "pdf", "vision"]) == ["document", "image", "text"]59 assert normalize_modalities("text, image and audio") == ["audio", "image", "text"]60 assert normalize_status("limited-availability") == "limited-availability" and normalize_status("Available") == "active"61 assert normalize_status("archived") == "archived" and normalize_status("sunset") == "deprecated" and normalize_status("shutdown") == "retired"62 assert normalize_hardware_kind("computer") == "system" and normalize_hardware_kind("soc") == "soc" and normalize_hardware_kind("GPU") == "gpu"63 assert normalize_org_kind("startup") == "company" and normalize_org_kind("research institute") == "lab" and normalize_org_kind("organization") is None646566def test_normalize_property_keeps_unknown_values() -> None:67 value, raw, maps = normalize_property("model", "license", "Apache 2.0")68 assert value == "Apache-2.0" and raw == "Apache 2.0" and ("license", "Apache 2.0", "Apache-2.0") in maps69 value, raw, maps = normalize_property("model", "license", "totally-custom")70 assert value == "totally-custom" and raw is None and maps == [("license", "totally-custom", None)]71 value, raw, maps = normalize_property("model", "modalities", ["Text", "smell", "pdf"])72 assert value == ["document", "smell", "text"] and raw == "Text, smell, pdf" # unknown modality kept, never dropped73 assert ("modality", "smell", None) in maps74 value, raw, _ = normalize_property("model", "modalities", ["text"])75 assert value == ["text"] and raw is None76 assert normalize_property("hardware", "kind", "computer")[0] == "system"77 assert normalize_property("framework", "kind", "framework")[0] == "training-framework"78 assert normalize_property("model", "kind", "whatever")[0] == "whatever" # `kind` is only a taxonomy for hardware/frameworks79 assert normalize_property("model", "parameter_count", 7_000_000_000) == (7_000_000_000, None, [])808182# ---------------------------------------------------------------------------------------------- model names83NAMES = {84 # name → (is_artifact, is_effort_variant, base_key, family_release_hint)85 "Qwen3.6-35B-A3B": (False, False, "qwen3-6-35b-a3b", "Qwen3.6"),86 "unsloth/Qwen3.6-35B-A3B-GGUF": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"),87 "Qwen3.6-35B-A3B-FP8": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"),88 "mlx-community/Kimi-K2.5": (True, False, "kimi-k2-5", "Kimi"),89 "zai-org/GLM-5-FP8": (True, False, "glm-5", "GLM5"),90 "amd/Llama-3.3-70B-Instruct-MXFP4": (True, False, "llama-3-3-70b-instruct", "Llama 3.3"),91 "bartowski/Qwen3.8-27B-GGUF": (True, False, "qwen3-8-27b", "Qwen3.8"),92 "meta-llama/Llama-4-Maverick-17B-128E-Instruct": (False, False, "llama-4-maverick-17b-128e-instruct", "Llama 4"),93 "claude-fable-5-1-xhigh": (False, True, "claude-fable-5-1", "Claude"),94 "gpt-6-astra-high": (False, True, "gpt-6-astra", "GPT 6"),95 "deepseek-v4-pro-0424-non-reasoning": (False, True, "deepseek-v4-pro-0424", "DeepSeek"),96 "gpt-5-4-mini-medium": (False, True, "gpt-5-4-mini", "GPT 5.4"),97 "Qwen3 Max Thinking": (False, True, "qwen3-max", "Qwen3"),98 "Kimi K2 Thinking": (False, False, "kimi-k2-thinking", "Kimi"),99 "Qwen3-Max": (False, False, "qwen3-max", "Qwen3"), # "max" is a tier, not an effort100 "GPT-5.1-Codex-Max": (False, False, "gpt-5-1-codex-max", "GPT 5.1"),101 "Grok 4.1 Fast": (False, False, "grok-4-1-fast", "Grok 4.1"),102 "gemini-2-0-flash-thinking-exp-1219": (False, False, "gemini-2-0-flash-thinking-exp-1219", "Gemini 2.0"),103 "Llama 3.1 70B Instruct": (False, False, "llama-3-1-70b-instruct", "Llama 3.1"),104 "Qwen 38B": (False, False, "qwen-38b", "Qwen"),105 "Qwen3-235B-A22B": (False, False, "qwen3-235b-a22b", "Qwen3"),106 "deepseek-ai/DeepSeek-V4-Flash-0731": (False, False, "deepseek-v4-flash-0731", "DeepSeek"),107}108109110@pytest.mark.parametrize("name", sorted(NAMES))111def test_analyze_model_name(name: str) -> None:112 is_artifact, is_variant, base, fam = NAMES[name]113 a = analyze_model_name(name)114 assert a.is_artifact is is_artifact, (name, a)115 assert a.is_effort_variant is is_variant, (name, a)116 assert a.base_key == base, (name, a.base_key)117 assert family_release_hint(name) == fam, (name, family_release_hint(name))118119120def test_variant_key_groups_artifacts_with_their_model() -> None:121 assert variant_key("Qwen3.6-35B-A3B") == variant_key("unsloth/Qwen3.6-35B-A3B-GGUF") == variant_key("Qwen3.6 35B A3B FP8") == "qwen3-6-35b-a3b"122 assert variant_key("Qwen3-8B") != variant_key("Qwen 38B")123 assert variant_key("Llama 3.1 70B Instruct") == variant_key("meta-llama/Llama-3.1-70B-Instruct") == "llama-3-1-70b"124125126def test_effort_helpers() -> None:127 assert base_name("gpt-5-4-mini-medium") == "gpt-5-4-mini" and base_name("Qwen3 Max Thinking") == "Qwen3 Max" and base_name("Qwen3-Max") == "Qwen3-Max"128 cfg = effort_config("gpt-5-4-mini-medium", {"aa_slug": "gpt-5-4-mini-medium", "evaluator": "AA"})129 assert cfg["reasoning_effort"] == "medium" and cfg["aa_variant_slug"] == "gpt-5-4-mini-medium" and cfg["evaluator"] == "AA"130 assert effort_config("Qwen3-Max", {"x": 1}) == {"x": 1}131 assert analyze_model_name("claude-fable-5-1-xhigh").parameter_count is None132 assert analyze_model_name("Qwen3.6-35B-A3B").active_parameter_count == 3_000_000_000133 assert family_hint("o3-mini") == "OpenAI o-series" and family_hint("random-thing") is None134 assert is_official_org("meta-llama", "Llama 4 Maverick") and not is_official_org("unsloth", "Llama 4 Maverick")135136137# ---------------------------------------------------------------------------------------------- benchmarks138def test_comparability_trio() -> None:139 a = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"}140 same = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"}141 cond = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "low"}142 task = {"variant": "Lite", "system": "mini-SWE-agent", "reasoning_effort": "high"}143 assert b.comparability(a, same, "resolved", "resolved")[0] == b.COMPARABLE144 assert b.comparability(a, cond, "resolved", "resolved")[0] == b.PARTIAL145 assert b.comparability(a, task, "resolved", "resolved")[0] == b.NOT_COMPARABLE146 assert b.comparability(a, same, "resolved", "pass@1")[0] == b.NOT_COMPARABLE147 assert b.comparability(a, same, same_benchmark=False)[0] == b.NOT_COMPARABLE148 # config_key ignores condition and bookkeeping keys, keeps task keys + metric149 assert b.config_key(a, "resolved") == b.config_key(cond, "resolved") == b.config_key({**a, "aa_slug": "x", "date": "2026"}, "resolved")150 assert b.config_key(a, "resolved") != b.config_key(task, "resolved") != b.config_key(a, "pass@1")151152153def test_metric_bounds_and_trust() -> None:154 assert b.metric_bounds("accuracy") == (0, 100) and b.metric_bounds("pass rate (2 attempts)") == (0, 100) and b.metric_bounds("category:Reasoning") == (0, 100)155 assert b.metric_bounds("elo") == (0, None) and b.metric_bounds("score") == (None, None) and b.metric_bounds("weird", "%") == (0, 100)156 assert b.normalize_metric("Percent Resolved") == "resolved" and b.normalize_metric("Elo / Bradley–Terry score") == "elo"157 assert b.trust_level("swebench.com", {"checked_by_swebench": True}) == "official-benchmark"158 assert b.trust_level("swebench.com", {"checked_by_swebench": False}) == "community"159 assert b.trust_level("artificialanalysis.ai") == "independent-evaluator" and b.trust_level("docs.claude.com") == "official-model-card"160 assert b.trust_level(None, extractor="llm") == "unverified"161 assert b.run_group_from_config({"release": "2026-06-25"}) == "2026-06-25" and b.run_group_from_config({"index_version": "4.3"}) == "4.3"162 assert b.variant_from_config({"board": "Multilingual"}) == "Multilingual" and b.family_of("swe-bench-verified") == ("swe-bench", "Verified")163