ontology: official family orgs, effort-config helpers, chained/budget effort suffixes, hyphenated family versions, property-level normalisation, run keys split from task keys
- models: FAMILY_ORGS / official_orgs / is_official_org; effort_config() and base_name() shared by the writer and the canonicalizer;
effort suffixes chain ("-thinking-64k-high-effort"), LiveBench budgets ("32k thinking", "(no thinking)"); "max"/"fast"/"instant" are model
tiers (Qwen3-Max, GPT-5.1-Codex-Max, Grok 4.1 Fast), no longer effort markers; family_release_hint reads "gpt-5-4" as 5.4 and never a size (38B)
- taxonomy: normalize_property(entity_type, prop, value) → (canonical, value_raw, mappings); unknown values kept, mapped to NULL
- benchmarks: RUN_KEYS (release, index_version, version, dataset_revision) leave config_key so a newer run replaces the older one; they are
condition keys for comparability (partially comparable)
- tests/test_ontology.py: licence table, openness derivation, taxonomy, 22 model names, comparability trio, metric bounds/trust
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +350 −21
modified
src/aiatlas/ontology/benchmarks.py
+10 −6
@@ -144,11 +144,13 @@ def trust_level(source_key: str | None, config: dict[str, Any] | None = None, *, | ||
| 144 | 144 | |
| 145 | 145 | # ---------------------------------------------------------------------------------------------- comparability |
| 146 | 146 | # Config keys that change the *task* (must match for full comparability). |
| 147 | −TASK_KEYS = ("variant", "board", "dataset_revision", "harness", "evaluator", "index_version", "release", "version", "subset", "split", "shots", | |
| 148 | − "pass_count", "attempts", "language", "scaffold", "agent", "system") | |
| 147 | +TASK_KEYS = ("variant", "board", "harness", "evaluator", "subset", "split", "shots", "pass_count", "attempts", "language", "scaffold", "agent", "system") | |
| 148 | +# Config keys that identify the *run* (a LiveBench release, an AA index version, a dataset revision): a newer run supersedes the older one | |
| 149 | +# for the same task (`run_group`, one current row per task), and two runs of the same task are only partially comparable. | |
| 150 | +RUN_KEYS = ("release", "index_version", "version", "dataset_revision") | |
| 149 | 151 | # Config keys that change the *conditions* (mismatch → partially comparable). |
| 150 | 152 | CONDITION_KEYS = ("reasoning_effort", "reasoning", "thinking_budget", "temperature", "judge", "tools", "tool_use", "max_tokens", "context_length", |
| 151 | − "sampling", "aggregation", "edit_format", "model_tag") | |
| 153 | + "sampling", "aggregation", "edit_format", "model_tag", *RUN_KEYS) | |
| 152 | 154 | # Keys that are pure bookkeeping (never affect comparability). |
| 153 | 155 | IGNORED_KEYS = {"aa_slug", "livebench_model_id", "api_model_id", "date", "submission", "checked_by_swebench", "open_source_system", "system_org", |
| 154 | 156 | "total_cost_usd", "cost_per_instance_usd", "dirname", "command", "versions", "test_cases", "seconds_per_case", "estimated", |
@@ -162,7 +164,8 @@ def _clean(v: Any) -> Any: | ||
| 162 | 164 | |
| 163 | 165 | |
| 164 | 166 | def config_key(config: dict[str, Any] | None, metric: str | None = None) -> str: |
| 165 | − """Stable hash of the comparability-relevant part of a result configuration (task keys + metric).""" | |
| 167 | + """Stable hash of the comparability-relevant part of a result configuration (task keys + metric). Run keys (release, index | |
| 168 | + version) are deliberately excluded: they define `run_group`, so the newest run of a task replaces the older ones on leaderboards.""" | |
| 166 | 169 | cfg = config or {} |
| 167 | 170 | core = {k: _clean(cfg[k]) for k in TASK_KEYS if k in cfg and cfg[k] not in (None, "", [], {})} |
| 168 | 171 | core["metric"] = normalize_metric(metric) or "" |
@@ -215,5 +218,6 @@ def run_group_from_config(config: dict[str, Any] | None) -> str | None: | ||
| 215 | 218 | return None |
| 216 | 219 | |
| 217 | 220 | |
| 218 | −__all__ = ["COMPARABLE", "CONDITION_KEYS", "FAMILIES", "IGNORED_KEYS", "METRICS", "NOT_COMPARABLE", "PARTIAL", "TASK_KEYS", "TRUST_LABELS", "TRUST_LEVELS", | |
| 219 | − "comparability", "config_key", "family_of", "metric_bounds", "normalize_metric", "run_group_from_config", "trust_level", "variant_from_config"] | |
| 221 | +__all__ = ["COMPARABLE", "CONDITION_KEYS", "FAMILIES", "IGNORED_KEYS", "METRICS", "NOT_COMPARABLE", "PARTIAL", "RUN_KEYS", "TASK_KEYS", "TRUST_LABELS", | |
| 222 | + "TRUST_LEVELS", "comparability", "config_key", "family_of", "metric_bounds", "normalize_metric", "run_group_from_config", "trust_level", | |
| 223 | + "variant_from_config"] | |
modified
src/aiatlas/ontology/models.py
+102 −13
@@ -43,12 +43,15 @@ CONVERTER_ORGS = { | ||
| 43 | 43 | EFFORT_SUFFIXES: dict[str, dict[str, str]] = { |
| 44 | 44 | "xhigh": {"reasoning_effort": "xhigh"}, "x-high": {"reasoning_effort": "xhigh"}, "extra-high": {"reasoning_effort": "xhigh"}, |
| 45 | 45 | "high": {"reasoning_effort": "high"}, "medium": {"reasoning_effort": "medium"}, "low": {"reasoning_effort": "low"}, "minimal": {"reasoning_effort": "minimal"}, |
| 46 | − "max": {"reasoning_effort": "max"}, "max-effort": {"reasoning_effort": "max"}, "high-effort": {"reasoning_effort": "high"}, "low-effort": {"reasoning_effort": "low"}, | |
| 46 | + "max-effort": {"reasoning_effort": "max"}, "high-effort": {"reasoning_effort": "high"}, "low-effort": {"reasoning_effort": "low"}, | |
| 47 | 47 | "medium-effort": {"reasoning_effort": "medium"}, "xhigh-effort": {"reasoning_effort": "xhigh"}, |
| 48 | 48 | "thinking": {"reasoning": "on"}, "reasoning": {"reasoning": "on"}, "think": {"reasoning": "on"}, "thinking-on": {"reasoning": "on"}, |
| 49 | 49 | "non-reasoning": {"reasoning": "off"}, "no-reasoning": {"reasoning": "off"}, "non-thinking": {"reasoning": "off"}, "nothink": {"reasoning": "off"}, |
| 50 | − "no-think": {"reasoning": "off"}, "instant": {"reasoning": "off"}, "thinking-off": {"reasoning": "off"}, "fast": {"reasoning": "off"}, | |
| 50 | + "no-think": {"reasoning": "off"}, "no-thinking": {"reasoning": "off"}, "thinking-off": {"reasoning": "off"}, "without-thinking": {"reasoning": "off"}, | |
| 51 | + "with-thinking": {"reasoning": "on"}, | |
| 51 | 52 | "adaptive": {"reasoning": "adaptive"}, "adaptive-reasoning": {"reasoning": "adaptive"}, |
| 53 | + # NOT effort suffixes although evaluators sometimes use them: "max" (Qwen3-Max, GPT-5.1-Codex-Max are models), "fast" (Grok 4.1 Fast), | |
| 54 | + # "instant" (Claude Instant) — they name a model tier, so they are only recognised in the explicit "-max-effort" form above. | |
| 52 | 55 | "thinking-16k": {"reasoning": "on", "thinking_budget": "16k"}, "thinking-32k": {"reasoning": "on", "thinking_budget": "32k"}, |
| 53 | 56 | "thinking-64k": {"reasoning": "on", "thinking_budget": "64k"}, "thinking-128k": {"reasoning": "on", "thinking_budget": "128k"}, |
| 54 | 57 | "thinking-8k": {"reasoning": "on", "thinking_budget": "8k"}, "thinking-4k": {"reasoning": "on", "thinking_budget": "4k"}, "thinking-1k": {"reasoning": "on", "thinking_budget": "1k"}, |
@@ -56,6 +59,10 @@ EFFORT_SUFFIXES: dict[str, dict[str, str]] = { | ||
| 56 | 59 | # order matters: try the longest compound suffixes first |
| 57 | 60 | _EFFORT_ORDERED = sorted(EFFORT_SUFFIXES, key=len, reverse=True) |
| 58 | 61 | _EFFORT_RE = re.compile(r"[-_ ](" + "|".join(re.escape(s) for s in _EFFORT_ORDERED) + r")$", re.I) |
| 62 | +# LiveBench-style budgets: "32k-thinking", "24k-think", "default-think(ing)" | |
| 63 | +_BUDGET_RE = re.compile(r"[-_ ]((\d+)k)[-_ ]?(thinking|think)$", re.I) | |
| 64 | +_DEFAULT_THINK_RE = re.compile(r"[-_ ](default[-_ ]?(thinking|think))$", re.I) | |
| 65 | +_MAX_EFFORT_SUFFIXES = 3 # "claude-opus-4-5-20251101-thinking-64k-high-effort" → thinking-64k + high-effort | |
| 59 | 66 | # Words that are *part of a model name*, never an effort suffix, when they precede the suffix (e.g. "Kimi K2 Thinking" is a distinct release). |
| 60 | 67 | OFFICIAL_THINKING_RELEASES = {"kimi-k2-thinking", "qwen3-235b-a22b-thinking-2507", "qwen3-30b-a3b-thinking-2507", "qwen3-4b-thinking-2507", "glm-4.5-air-thinking", |
| 61 | 68 | "gemini-2.5-flash-thinking", "grok-3-mini-thinking", "gemini-2-0-flash-thinking-exp-1219", "gemini-2-0-flash-thinking-exp-01-21"} |
@@ -116,13 +123,33 @@ def analyze_model_name(raw: str) -> NameAnalysis: | ||
| 116 | 123 | low = re.sub(r"[()\[\]]+", "-", low).strip("-") |
| 117 | 124 | low = re.sub(r"-{2,}", "-", low) |
| 118 | 125 | |
| 119 | − # evaluator effort suffix (only when the remaining stem is not itself an official "Thinking" release) | |
| 120 | − m = _EFFORT_RE.search(low) | |
| 121 | − if m and low not in OFFICIAL_THINKING_RELEASES: | |
| 122 | − suffix = m.group(1).lower() | |
| 123 | − a.effort = dict(EFFORT_SUFFIXES[suffix]) | |
| 124 | − a.effort_suffix = suffix | |
| 125 | − low = low[: m.start()] | |
| 126 | + # evaluator effort suffixes (up to three, e.g. "-thinking-64k-high-effort"), only when the stem is not itself an official "Thinking" release | |
| 127 | + stripped: list[str] = [] | |
| 128 | + for _ in range(_MAX_EFFORT_SUFFIXES): | |
| 129 | + if low in OFFICIAL_THINKING_RELEASES: | |
| 130 | + break | |
| 131 | + m = _BUDGET_RE.search(low) # "-32k-thinking" before the bare "-thinking" | |
| 132 | + if m: | |
| 133 | + a.effort = {"reasoning": "on", "thinking_budget": m.group(1).lower(), **a.effort} | |
| 134 | + stripped.insert(0, m.group(0)[1:].lower()) | |
| 135 | + low = low[: m.start()] | |
| 136 | + continue | |
| 137 | + m = _DEFAULT_THINK_RE.search(low) | |
| 138 | + if m: | |
| 139 | + a.effort = {"reasoning": "on", **a.effort} | |
| 140 | + stripped.insert(0, m.group(1).lower()) | |
| 141 | + low = low[: m.start()] | |
| 142 | + continue | |
| 143 | + m = _EFFORT_RE.search(low) | |
| 144 | + if m: | |
| 145 | + suffix = m.group(1).lower() | |
| 146 | + a.effort = {**EFFORT_SUFFIXES[suffix], **a.effort} | |
| 147 | + stripped.insert(0, suffix) | |
| 148 | + low = low[: m.start()] | |
| 149 | + continue | |
| 150 | + break | |
| 151 | + if stripped: | |
| 152 | + a.effort_suffix = "-".join(stripped) | |
| 126 | 153 | |
| 127 | 154 | # sizes |
| 128 | 155 | am = _ACTIVE_RE.search(low) |
@@ -252,10 +279,13 @@ def family_release_hint(name: str) -> str | None: | ||
| 252 | 279 | if not fam: |
| 253 | 280 | return None |
| 254 | 281 | n = name.split("/")[-1] |
| 255 | − m = re.search(re.escape(fam.split(" ")[0]) + r"[\s\-]?(\d+(?:\.\d+)?)", n, re.I) | |
| 282 | + # version = digits right after the family word, optionally ".minor" or "-minor" (AA/OpenRouter slugs write 5.4 as 5-4); | |
| 283 | + # a number followed by a size unit (38B, 235B, 1.5T) is a parameter count, never a version. | |
| 284 | + m = re.search(re.escape(fam.split(" ")[0]) + r"[\s\-]?(\d+)(?![0-9]*\.?\d*[bmt](?![a-z]))(?:[.\-](\d+)(?![0-9]*[bmt](?![a-z])))?(?![0-9])", n, re.I) | |
| 256 | 285 | if m: |
| 286 | + version = m.group(1) + (f".{m.group(2)}" if m.group(2) else "") | |
| 257 | 287 | joined = fam in ("Qwen", "GLM", "Phi", "Yi", "Step") or re.search(re.escape(fam) + r"\d", n, re.I) |
| 258 | − return f"{fam}{m.group(1)}" if joined else f"{fam} {m.group(1)}" | |
| 288 | + return f"{fam}{version}" if joined else f"{fam} {version}" | |
| 259 | 289 | return fam |
| 260 | 290 | |
| 261 | 291 | |
@@ -267,5 +297,64 @@ def variant_key(name: str) -> str: | ||
| 267 | 297 | return "-".join(toks) |
| 268 | 298 | |
| 269 | 299 | |
| 270 | −__all__ = ["ARTIFACT_PACKAGING", "CONVERTER_ORGS", "EFFORT_SUFFIXES", "NameAnalysis", "PRECISION_FORMATS", "QUANT_FORMATS", "analyze_model_name", | |
| 271 | − "family_hint", "family_release_hint", "variant_key"] | |
| 300 | +# ---------------------------------------------------------------------------------------------- official organisations per family | |
| 301 | +# family root (lowercase `family_hint`) → organisation slugs / hub org names (lowercase) that publish the *official* checkpoints. | |
| 302 | +# A repository under one of these orgs is the model (or its official checkpoint), never a third-party artifact. | |
| 303 | +FAMILY_ORGS: dict[str, tuple[str, ...]] = { | |
| 304 | + "llama": ("meta-llama", "meta", "meta-ai", "facebook"), "qwen": ("qwen", "alibaba", "alibaba-cloud", "alibaba-qwen"), "gemma": ("google", "google-deepmind"), | |
| 305 | + "gemini": ("google", "google-deepmind"), "palm": ("google",), "claude": ("anthropic",), "gpt": ("openai",), "gpt-oss": ("openai",), "openai o-series": ("openai",), | |
| 306 | + "whisper": ("openai",), "dall·e": ("openai",), "sora": ("openai",), "deepseek": ("deepseek-ai", "deepseek"), "kimi": ("moonshotai", "moonshot-ai", "moonshot"), | |
| 307 | + "glm": ("zai-org", "thudm", "zhipu-ai", "z-ai", "zhipu"), "mistral": ("mistralai", "mistral-ai", "mistral"), "mixtral": ("mistralai", "mistral-ai", "mistral"), | |
| 308 | + "ministral": ("mistralai", "mistral-ai", "mistral"), "codestral": ("mistralai", "mistral-ai", "mistral"), "devstral": ("mistralai", "mistral-ai", "mistral"), | |
| 309 | + "magistral": ("mistralai", "mistral-ai", "mistral"), "pixtral": ("mistralai", "mistral-ai", "mistral"), "voxtral": ("mistralai", "mistral-ai", "mistral"), | |
| 310 | + "phi": ("microsoft",), "grok": ("xai", "x-ai", "xai-org"), "command": ("cohereforai", "cohere", "coherelabs", "cohere-labs"), "aya": ("cohereforai", "cohere", "coherelabs"), | |
| 311 | + "nemotron": ("nvidia",), "granite": ("ibm-granite", "ibm"), "olmo": ("allenai", "ai2"), "molmo": ("allenai", "ai2"), "falcon": ("tiiuae", "tii"), "yi": ("01-ai",), | |
| 312 | + "minimax": ("minimaxai", "minimax"), "hunyuan": ("tencent", "tencent-hunyuan"), "ernie": ("baidu",), "seed": ("bytedance-seed", "bytedance"), "doubao": ("bytedance",), | |
| 313 | + "step": ("stepfun-ai", "stepfun"), "internlm": ("internlm", "shanghai-ai-laboratory", "opengvlab"), "jamba": ("ai21labs", "ai21"), "lfm": ("liquidai", "liquid-ai"), | |
| 314 | + "exaone": ("lgai-exaone", "lg-ai-research"), "solar": ("upstage",), "nova": ("amazon", "aws"), "titan": ("amazon", "aws"), "sonar": ("perplexity-ai", "perplexity"), | |
| 315 | + "stable diffusion": ("stabilityai", "stability-ai"), "flux": ("black-forest-labs",), "veo": ("google",), "imagen": ("google",), "cogito": ("deepcogito",), | |
| 316 | + "hermes": ("nousresearch",), "smollm": ("huggingfacetb", "hugging-face"), "bert": ("google", "google-bert"), "t5": ("google", "google-t5"), "clip": ("openai",), | |
| 317 | +} | |
| 318 | + | |
| 319 | + | |
| 320 | +def official_orgs(name: str) -> tuple[str, ...]: | |
| 321 | + """Official organisation slugs for the family the name belongs to (empty when unknown).""" | |
| 322 | + fam = family_hint(name) | |
| 323 | + return FAMILY_ORGS.get(fam.lower(), ()) if fam else () | |
| 324 | + | |
| 325 | + | |
| 326 | +def is_official_org(repo_org: str | None, name: str) -> bool: | |
| 327 | + return bool(repo_org) and repo_org.lower() in official_orgs(name) | |
| 328 | + | |
| 329 | + | |
| 330 | +def effort_config(name: str, config: dict | None) -> dict: | |
| 331 | + """Result configuration for an evaluation-effort variant: the effort dict (`reasoning_effort`, `reasoning`, `thinking_budget`) is merged | |
| 332 | + into the config and the evaluator's variant slug is kept as `aa_variant_slug` so folded results stay distinguishable. Names that are | |
| 333 | + not effort variants return the config unchanged.""" | |
| 334 | + cfg = dict(config or {}) | |
| 335 | + a = analyze_model_name(name) | |
| 336 | + if not a.is_effort_variant: | |
| 337 | + return cfg | |
| 338 | + for k, v in a.effort.items(): | |
| 339 | + cfg.setdefault(k, v) | |
| 340 | + slug = cfg.get("aa_slug") or a.raw.split("/")[-1].strip().lower().replace(" ", "-") | |
| 341 | + cfg.setdefault("aa_variant_slug", slug) | |
| 342 | + return cfg | |
| 343 | + | |
| 344 | + | |
| 345 | +def base_name(name: str) -> str: | |
| 346 | + """The name with the evaluator effort suffix removed ("gpt-5-4-mini-medium" → "gpt-5-4-mini", "Claude Sonnet 4 (no thinking)" → | |
| 347 | + "Claude Sonnet 4"); unchanged when not a variant.""" | |
| 348 | + a = analyze_model_name(name) | |
| 349 | + if not a.is_effort_variant or not a.effort_suffix: | |
| 350 | + return name.strip() | |
| 351 | + n = name.strip() | |
| 352 | + # the suffix was detected on a normalised form (spaces/underscores/parentheses → "-"): strip it from the raw name with the same tolerance | |
| 353 | + tokens = [re.escape(t) for t in a.effort_suffix.split("-") if t] | |
| 354 | + pattern = r"[\s\-_(]+" + r"[\s\-_]*".join(tokens) + r"[)\s]*$" | |
| 355 | + out = re.sub(pattern, "", n, flags=re.I) | |
| 356 | + return out.strip().rstrip("-_ (").strip() or n | |
| 357 | + | |
| 358 | + | |
| 359 | +__all__ = ["ARTIFACT_PACKAGING", "CONVERTER_ORGS", "EFFORT_SUFFIXES", "FAMILY_ORGS", "NameAnalysis", "PRECISION_FORMATS", "QUANT_FORMATS", "analyze_model_name", | |
| 360 | + "base_name", "effort_config", "family_hint", "family_release_hint", "is_official_org", "official_orgs", "variant_key"] | |
modified
src/aiatlas/ontology/taxonomy.py
+76 −2
@@ -149,6 +149,79 @@ def normalize_framework_kind(raw: Any) -> str | None: | ||
| 149 | 149 | return _FW_ALIASES.get(s) or _FW_ALIASES.get(s.replace("_", "-")) or _FW_ALIASES.get(s.replace("-", " ")) |
| 150 | 150 | |
| 151 | 151 | |
| 152 | +# ---------------------------------------------------------------------------------------------- property-level normalisation (writer + canonicalizer) | |
| 153 | +# property → taxonomy domain (the entity type picks the normaliser for `kind`) | |
| 154 | +TAXONOMY_PROPERTIES = {"license", "openness", "modalities", "modalities_input", "modalities_output", "status", "kind", "org_kind"} | |
| 155 | +_ORG_TYPES = {"company", "organization", "lab", "university", "provider"} | |
| 156 | +_FRAMEWORK_TYPES = {"framework", "library", "tool", "runtime", "agent", "mcp_server", "repository"} | |
| 157 | + | |
| 158 | + | |
| 159 | +def normalize_property(entity_type: str | None, prop: str, value: Any) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]: | |
| 160 | + """Canonicalise one attribute value at write time. | |
| 161 | + | |
| 162 | + Returns `(canonical_value, value_raw, mappings)` where `value_raw` is the source string when it differed from the canonical value | |
| 163 | + (None otherwise) and `mappings` lists `(domain, raw, canonical|None)` pairs observed — unknown values are KEPT AS-IS (never dropped) | |
| 164 | + and mapped to canonical=None so the taxonomy backlog stays visible.""" | |
| 165 | + if value is None or prop not in TAXONOMY_PROPERTIES: | |
| 166 | + return value, None, [] | |
| 167 | + if prop == "license": | |
| 168 | + from aiatlas.ontology.licenses import normalize_license | |
| 169 | + | |
| 170 | + return _scalar("license", value, normalize_license) | |
| 171 | + if prop == "openness": | |
| 172 | + from aiatlas.ontology.openness import normalize_openness | |
| 173 | + | |
| 174 | + return _scalar("openness", value, normalize_openness) | |
| 175 | + if prop == "status": | |
| 176 | + return _scalar("status", value, normalize_status) | |
| 177 | + if prop == "org_kind": | |
| 178 | + return _scalar("org_kind", value, normalize_org_kind) | |
| 179 | + if prop == "kind": | |
| 180 | + if entity_type == "hardware": | |
| 181 | + return _scalar("hardware_kind", value, normalize_hardware_kind) | |
| 182 | + if entity_type in _FRAMEWORK_TYPES: | |
| 183 | + return _scalar("framework_kind", value, normalize_framework_kind) | |
| 184 | + return value, None, [] | |
| 185 | + if prop in ("modalities", "modalities_input", "modalities_output"): | |
| 186 | + items = value if isinstance(value, (list, tuple, set)) else re.split(r"[,/;+&]|\band\b|→|->", str(value)) | |
| 187 | + out: list[str] = [] | |
| 188 | + mappings: list[tuple[str, str, str | None]] = [] | |
| 189 | + changed = False | |
| 190 | + for it in items: | |
| 191 | + raw = str(it).strip() | |
| 192 | + if not raw: | |
| 193 | + continue | |
| 194 | + canon = normalize_modality(raw) | |
| 195 | + if canon is None and raw.lower() in _MODALITY_ALIASES: # "multimodal" & co: a property of the set, not a modality | |
| 196 | + mappings.append(("modality", raw, None)) | |
| 197 | + changed = True | |
| 198 | + continue | |
| 199 | + mappings.append(("modality", raw, canon)) | |
| 200 | + keep = canon or raw | |
| 201 | + if keep != raw: | |
| 202 | + changed = True | |
| 203 | + if keep not in out: | |
| 204 | + out.append(keep) | |
| 205 | + canonical = sorted(out) | |
| 206 | + if not isinstance(value, (list, tuple, set)): | |
| 207 | + changed = True | |
| 208 | + elif sorted(str(x).strip() for x in value if str(x).strip()) != canonical: | |
| 209 | + changed = True | |
| 210 | + raw_repr = (", ".join(str(x) for x in value) if isinstance(value, (list, tuple, set)) else str(value)) if changed else None | |
| 211 | + return canonical, raw_repr, mappings | |
| 212 | + return value, None, [] | |
| 213 | + | |
| 214 | + | |
| 215 | +def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]: | |
| 216 | + if not isinstance(value, str): | |
| 217 | + return value, None, [] | |
| 218 | + raw = value.strip() | |
| 219 | + canon = normaliser(raw) | |
| 220 | + if canon is None: | |
| 221 | + return raw, None, [(domain, raw, None)] | |
| 222 | + return canon, (raw if raw != canon else None), [(domain, raw, canon)] | |
| 223 | + | |
| 224 | + | |
| 152 | 225 | # ---------------------------------------------------------------------------------------------- generic helper |
| 153 | 226 | def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]: |
| 154 | 227 | """Return (canonical_or_original, raw_if_changed). Lets the writer store `x` canonical and `x_raw` when the source label differed.""" |
@@ -161,6 +234,7 @@ def canonical_enum(value: Any, normaliser) -> tuple[Any, str | None]: | ||
| 161 | 234 | |
| 162 | 235 | |
| 163 | 236 | __all__ = [ |
| 164 | − "FRAMEWORK_KINDS", "HARDWARE_KINDS", "MODALITIES", "MODEL_STATUSES", "ORG_KINDS", "ORG_TYPE_DEFAULT_KIND", "canonical_enum", | |
| 165 | − "normalize_framework_kind", "normalize_hardware_kind", "normalize_modalities", "normalize_modality", "normalize_org_kind", "normalize_status", | |
| 237 | + "FRAMEWORK_KINDS", "HARDWARE_KINDS", "MODALITIES", "MODEL_STATUSES", "ORG_KINDS", "ORG_TYPE_DEFAULT_KIND", "TAXONOMY_PROPERTIES", "canonical_enum", | |
| 238 | + "normalize_framework_kind", "normalize_hardware_kind", "normalize_modalities", "normalize_modality", "normalize_org_kind", "normalize_property", | |
| 239 | + "normalize_status", | |
| 166 | 240 | ] |
added
tests/test_ontology.py
+162 −0
@@ -0,0 +1,162 @@ | ||
| 1 | +"""Ontology — pure functions, no database.""" | |
| 2 | +from __future__ import annotations | |
| 3 | + | |
| 4 | +import pytest | |
| 5 | + | |
| 6 | +from aiatlas.ontology import benchmarks as b | |
| 7 | +from aiatlas.ontology.licenses import LICENSES, license_info, normalize_license | |
| 8 | +from aiatlas.ontology.models import analyze_model_name, base_name, effort_config, family_hint, family_release_hint, is_official_org, variant_key | |
| 9 | +from aiatlas.ontology.openness import derive_openness, normalize_openness, openness_dimensions | |
| 10 | +from aiatlas.ontology.taxonomy import normalize_hardware_kind, normalize_modalities, normalize_org_kind, normalize_property, normalize_status | |
| 11 | + | |
| 12 | + | |
| 13 | +# ---------------------------------------------------------------------------------------------- licences | |
| 14 | +@pytest.mark.parametrize("raw", ["Apache 2.0", "apache-2.0", "Apache-2.0", "Apache License 2.0", "apache license, version 2.0", "APACHE2", "License: apache-2.0"]) | |
| 15 | +def test_apache_variants_one_key(raw: str) -> None: | |
| 16 | + assert normalize_license(raw) == "Apache-2.0" | |
| 17 | + | |
| 18 | + | |
| 19 | +@pytest.mark.parametrize("raw,key", [ | |
| 20 | + ("mit", "MIT"), ("Modified MIT", "MIT-Modified"), ("CC BY-NC 4.0", "CC-BY-NC-4.0"), ("cc-by-nc-4.0", "CC-BY-NC-4.0"), ("cc-by-4.0", "CC-BY-4.0"), | |
| 21 | + ("llama3", "Llama-3-Community"), ("llama3.1", "Llama-3.1-Community"), ("llama3.2", "Llama-3.2-Community"), ("Llama 4 Community License", "Llama-4-Community"), | |
| 22 | + ("gemma", "Gemma-Terms"), ("openrail++", "OpenRAIL++-M"), ("bigscience-bloom-rail-1.0", "BigScience-BLOOM-RAIL-1.0"), ("other", "Other"), | |
| 23 | + ("apple-amlr", "Apple-AMLR"), ("bsd-3-clause", "BSD-3-Clause"), ("proprietary", "Proprietary"), ("GPLv3", "GPL-3.0"), | |
| 24 | +]) | |
| 25 | +def test_license_table(raw: str, key: str) -> None: | |
| 26 | + assert normalize_license(raw) == key | |
| 27 | + assert key in LICENSES | |
| 28 | + | |
| 29 | + | |
| 30 | +def test_unknown_license_is_none_not_guessed() -> None: | |
| 31 | + assert normalize_license("weird-license-xyz") is None | |
| 32 | + assert normalize_license("") is None and normalize_license(None) is None | |
| 33 | + assert license_info("weird") is None | |
| 34 | + | |
| 35 | + | |
| 36 | +# ---------------------------------------------------------------------------------------------- openness | |
| 37 | +def test_openness_derivation() -> None: | |
| 38 | + llama = openness_dimensions(weights_available=True, license_key="Llama-3.1-Community") | |
| 39 | + assert derive_openness(llama, license_key="Llama-3.1-Community") == "restricted-weights" | |
| 40 | + apache = openness_dimensions(weights_available=True, license_key="Apache-2.0") | |
| 41 | + assert derive_openness(apache, license_key="Apache-2.0") == "open-weights" | |
| 42 | + assert derive_openness({**apache, "source_code_available": True}, license_key="Apache-2.0") == "open-source" | |
| 43 | + prop = openness_dimensions(weights_available=None, license_key="Proprietary") | |
| 44 | + assert prop["weights_available"] is False and derive_openness(prop, license_key="Proprietary") == "proprietary" | |
| 45 | + assert derive_openness(openness_dimensions(weights_available=None), license_key=None) == "unknown" | |
| 46 | + nc = openness_dimensions(weights_available=True, license_key="CC-BY-NC-4.0") | |
| 47 | + assert nc["commercial_use_allowed"] is False and derive_openness(nc, license_key="CC-BY-NC-4.0") == "restricted-weights" | |
| 48 | + | |
| 49 | + | |
| 50 | +@pytest.mark.parametrize("raw,canon", [("open-weights", "open-weights"), ("restricted", "restricted-weights"), ("gated", "restricted-weights"), ("closed", "proprietary"), | |
| 51 | + ("Open Source", "open-source"), ("", "unknown"), ("banana", None)]) | |
| 52 | +def test_normalize_openness(raw: str, canon: str | None) -> None: | |
| 53 | + assert normalize_openness(raw) == canon | |
| 54 | + | |
| 55 | + | |
| 56 | +# ---------------------------------------------------------------------------------------------- taxonomy | |
| 57 | +def test_modalities_and_status_and_kinds() -> None: | |
| 58 | + assert normalize_modalities(["Text", "text", "pdf", "vision"]) == ["document", "image", "text"] | |
| 59 | + assert normalize_modalities("text, image and audio") == ["audio", "image", "text"] | |
| 60 | + assert normalize_status("limited-availability") == "limited-availability" and normalize_status("Available") == "active" | |
| 61 | + assert normalize_status("archived") == "archived" and normalize_status("sunset") == "deprecated" and normalize_status("shutdown") == "retired" | |
| 62 | + assert normalize_hardware_kind("computer") == "system" and normalize_hardware_kind("soc") == "soc" and normalize_hardware_kind("GPU") == "gpu" | |
| 63 | + assert normalize_org_kind("startup") == "company" and normalize_org_kind("research institute") == "lab" and normalize_org_kind("organization") is None | |
| 64 | + | |
| 65 | + | |
| 66 | +def test_normalize_property_keeps_unknown_values() -> None: | |
| 67 | + value, raw, maps = normalize_property("model", "license", "Apache 2.0") | |
| 68 | + assert value == "Apache-2.0" and raw == "Apache 2.0" and ("license", "Apache 2.0", "Apache-2.0") in maps | |
| 69 | + value, raw, maps = normalize_property("model", "license", "totally-custom") | |
| 70 | + assert value == "totally-custom" and raw is None and maps == [("license", "totally-custom", None)] | |
| 71 | + value, raw, maps = normalize_property("model", "modalities", ["Text", "smell", "pdf"]) | |
| 72 | + assert value == ["document", "smell", "text"] and raw == "Text, smell, pdf" # unknown modality kept, never dropped | |
| 73 | + assert ("modality", "smell", None) in maps | |
| 74 | + value, raw, _ = normalize_property("model", "modalities", ["text"]) | |
| 75 | + assert value == ["text"] and raw is None | |
| 76 | + assert normalize_property("hardware", "kind", "computer")[0] == "system" | |
| 77 | + assert normalize_property("framework", "kind", "framework")[0] == "training-framework" | |
| 78 | + assert normalize_property("model", "kind", "whatever")[0] == "whatever" # `kind` is only a taxonomy for hardware/frameworks | |
| 79 | + assert normalize_property("model", "parameter_count", 7_000_000_000) == (7_000_000_000, None, []) | |
| 80 | + | |
| 81 | + | |
| 82 | +# ---------------------------------------------------------------------------------------------- model names | |
| 83 | +NAMES = { | |
| 84 | + # name → (is_artifact, is_effort_variant, base_key, family_release_hint) | |
| 85 | + "Qwen3.6-35B-A3B": (False, False, "qwen3-6-35b-a3b", "Qwen3.6"), | |
| 86 | + "unsloth/Qwen3.6-35B-A3B-GGUF": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"), | |
| 87 | + "Qwen3.6-35B-A3B-FP8": (True, False, "qwen3-6-35b-a3b", "Qwen3.6"), | |
| 88 | + "mlx-community/Kimi-K2.5": (True, False, "kimi-k2-5", "Kimi"), | |
| 89 | + "zai-org/GLM-5-FP8": (True, False, "glm-5", "GLM5"), | |
| 90 | + "amd/Llama-3.3-70B-Instruct-MXFP4": (True, False, "llama-3-3-70b-instruct", "Llama 3.3"), | |
| 91 | + "bartowski/Qwen3.8-27B-GGUF": (True, False, "qwen3-8-27b", "Qwen3.8"), | |
| 92 | + "meta-llama/Llama-4-Maverick-17B-128E-Instruct": (False, False, "llama-4-maverick-17b-128e-instruct", "Llama 4"), | |
| 93 | + "claude-fable-5-1-xhigh": (False, True, "claude-fable-5-1", "Claude"), | |
| 94 | + "gpt-6-astra-high": (False, True, "gpt-6-astra", "GPT 6"), | |
| 95 | + "deepseek-v4-pro-0424-non-reasoning": (False, True, "deepseek-v4-pro-0424", "DeepSeek"), | |
| 96 | + "gpt-5-4-mini-medium": (False, True, "gpt-5-4-mini", "GPT 5.4"), | |
| 97 | + "Qwen3 Max Thinking": (False, True, "qwen3-max", "Qwen3"), | |
| 98 | + "Kimi K2 Thinking": (False, False, "kimi-k2-thinking", "Kimi"), | |
| 99 | + "Qwen3-Max": (False, False, "qwen3-max", "Qwen3"), # "max" is a tier, not an effort | |
| 100 | + "GPT-5.1-Codex-Max": (False, False, "gpt-5-1-codex-max", "GPT 5.1"), | |
| 101 | + "Grok 4.1 Fast": (False, False, "grok-4-1-fast", "Grok 4.1"), | |
| 102 | + "gemini-2-0-flash-thinking-exp-1219": (False, False, "gemini-2-0-flash-thinking-exp-1219", "Gemini 2.0"), | |
| 103 | + "Llama 3.1 70B Instruct": (False, False, "llama-3-1-70b-instruct", "Llama 3.1"), | |
| 104 | + "Qwen 38B": (False, False, "qwen-38b", "Qwen"), | |
| 105 | + "Qwen3-235B-A22B": (False, False, "qwen3-235b-a22b", "Qwen3"), | |
| 106 | + "deepseek-ai/DeepSeek-V4-Flash-0731": (False, False, "deepseek-v4-flash-0731", "DeepSeek"), | |
| 107 | +} | |
| 108 | + | |
| 109 | + | |
| 110 | +@pytest.mark.parametrize("name", sorted(NAMES)) | |
| 111 | +def test_analyze_model_name(name: str) -> None: | |
| 112 | + is_artifact, is_variant, base, fam = NAMES[name] | |
| 113 | + a = analyze_model_name(name) | |
| 114 | + assert a.is_artifact is is_artifact, (name, a) | |
| 115 | + assert a.is_effort_variant is is_variant, (name, a) | |
| 116 | + assert a.base_key == base, (name, a.base_key) | |
| 117 | + assert family_release_hint(name) == fam, (name, family_release_hint(name)) | |
| 118 | + | |
| 119 | + | |
| 120 | +def test_variant_key_groups_artifacts_with_their_model() -> None: | |
| 121 | + assert variant_key("Qwen3.6-35B-A3B") == variant_key("unsloth/Qwen3.6-35B-A3B-GGUF") == variant_key("Qwen3.6 35B A3B FP8") == "qwen3-6-35b-a3b" | |
| 122 | + assert variant_key("Qwen3-8B") != variant_key("Qwen 38B") | |
| 123 | + assert variant_key("Llama 3.1 70B Instruct") == variant_key("meta-llama/Llama-3.1-70B-Instruct") == "llama-3-1-70b" | |
| 124 | + | |
| 125 | + | |
| 126 | +def test_effort_helpers() -> None: | |
| 127 | + assert base_name("gpt-5-4-mini-medium") == "gpt-5-4-mini" and base_name("Qwen3 Max Thinking") == "Qwen3 Max" and base_name("Qwen3-Max") == "Qwen3-Max" | |
| 128 | + cfg = effort_config("gpt-5-4-mini-medium", {"aa_slug": "gpt-5-4-mini-medium", "evaluator": "AA"}) | |
| 129 | + assert cfg["reasoning_effort"] == "medium" and cfg["aa_variant_slug"] == "gpt-5-4-mini-medium" and cfg["evaluator"] == "AA" | |
| 130 | + assert effort_config("Qwen3-Max", {"x": 1}) == {"x": 1} | |
| 131 | + assert analyze_model_name("claude-fable-5-1-xhigh").parameter_count is None | |
| 132 | + assert analyze_model_name("Qwen3.6-35B-A3B").active_parameter_count == 3_000_000_000 | |
| 133 | + assert family_hint("o3-mini") == "OpenAI o-series" and family_hint("random-thing") is None | |
| 134 | + assert is_official_org("meta-llama", "Llama 4 Maverick") and not is_official_org("unsloth", "Llama 4 Maverick") | |
| 135 | + | |
| 136 | + | |
| 137 | +# ---------------------------------------------------------------------------------------------- benchmarks | |
| 138 | +def test_comparability_trio() -> None: | |
| 139 | + a = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"} | |
| 140 | + same = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "high"} | |
| 141 | + cond = {"variant": "Verified", "system": "mini-SWE-agent", "reasoning_effort": "low"} | |
| 142 | + task = {"variant": "Lite", "system": "mini-SWE-agent", "reasoning_effort": "high"} | |
| 143 | + assert b.comparability(a, same, "resolved", "resolved")[0] == b.COMPARABLE | |
| 144 | + assert b.comparability(a, cond, "resolved", "resolved")[0] == b.PARTIAL | |
| 145 | + assert b.comparability(a, task, "resolved", "resolved")[0] == b.NOT_COMPARABLE | |
| 146 | + assert b.comparability(a, same, "resolved", "pass@1")[0] == b.NOT_COMPARABLE | |
| 147 | + assert b.comparability(a, same, same_benchmark=False)[0] == b.NOT_COMPARABLE | |
| 148 | + # config_key ignores condition and bookkeeping keys, keeps task keys + metric | |
| 149 | + assert b.config_key(a, "resolved") == b.config_key(cond, "resolved") == b.config_key({**a, "aa_slug": "x", "date": "2026"}, "resolved") | |
| 150 | + assert b.config_key(a, "resolved") != b.config_key(task, "resolved") != b.config_key(a, "pass@1") | |
| 151 | + | |
| 152 | + | |
| 153 | +def test_metric_bounds_and_trust() -> None: | |
| 154 | + assert b.metric_bounds("accuracy") == (0, 100) and b.metric_bounds("pass rate (2 attempts)") == (0, 100) and b.metric_bounds("category:Reasoning") == (0, 100) | |
| 155 | + assert b.metric_bounds("elo") == (0, None) and b.metric_bounds("score") == (None, None) and b.metric_bounds("weird", "%") == (0, 100) | |
| 156 | + assert b.normalize_metric("Percent Resolved") == "resolved" and b.normalize_metric("Elo / Bradley–Terry score") == "elo" | |
| 157 | + assert b.trust_level("swebench.com", {"checked_by_swebench": True}) == "official-benchmark" | |
| 158 | + assert b.trust_level("swebench.com", {"checked_by_swebench": False}) == "community" | |
| 159 | + assert b.trust_level("artificialanalysis.ai") == "independent-evaluator" and b.trust_level("docs.claude.com") == "official-model-card" | |
| 160 | + assert b.trust_level(None, extractor="llm") == "unverified" | |
| 161 | + assert b.run_group_from_config({"release": "2026-06-25"}) == "2026-06-25" and b.run_group_from_config({"index_version": "4.3"}) == "4.3" | |
| 162 | + assert b.variant_from_config({"board": "Multilingual"}) == "Multilingual" and b.family_of("swe-bench-verified") == ("swe-bench", "Verified") | |
| 163 | ||