HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Public leaderboards → append-only `benchmark_results` (tier 2). One connector class per leaderboard domain (the SDK binds one2`source_key` per connector, and each board has its own rate limit and provenance) — module kept as `leaderboards`.34 * aider_leaderboard raw polyglot_leaderboard.yml from the aider repository → `aider-polyglot` (pass_rate_2) and the separate5 `aider-polyglot-well-formed` benchmark (percent_cases_well_formed); run group = run date.6 * swebench_leaderboard https://www.swebench.com/ embeds `<script id="leaderboard-data">` (5 boards: Verified, Lite, Test, Multimodal,7 Multilingual) → results keyed by (model, benchmark, config {system, system_org, model_tag, reasoning_effort…});8 trust `official-benchmark` when SWE-bench checked the submission, `community` otherwise.9 * livebench_leaderboard https://livebench.ai/ is a React app; its bundle lists the release dates and fetches `./table_<date>.csv` +10 `./categories_<date>.json` (plus a model metadata map with display names / HF links). The connector follows11 index → bundle → categories → table and emits the overall mean on `livebench` and each category average on its12 own benchmark entity (`livebench-<category>`, family livebench) — never mixed metrics on one board.13 * artificial_analysis https://artificialanalysis.ai/leaderboards/models — Next.js RSC flight payload (`self.__next_f.push`) with the full14 model list: Intelligence Index (config {version}) + component evaluations mapped to registry benchmarks.1516Model identity on leaderboards (`connectors/_identity.py`): names are free text or evaluator slugs → `EntityRef("model", …,17identity_confidence="medium")` with resolver-friendly aliases and the developer organisation when the id/name is first-party.18**Evaluation-effort variants are configurations, not models**: `claude-opus-5-xhigh`, "GPT-5.5 (xhigh)", `…-thinking-64k-high-effort`19all point at the base model with `config.reasoning_effort` / `config.reasoning` / `config.thinking_budget` set.20"""21from __future__ import annotations2223import csv24import io25import json26import re27from datetime import UTC, date, datetime28from statistics import fmean29from typing import Any3031import yaml3233from aiatlas.connectors._identity import (34 family_ref,35 model_identity,36 model_ref_from_api_id,37 org_key_for_bare_id,38 org_ref_in,39 split_effort_label,40 strip_effort,41 strip_effort_words,42)43from aiatlas.ontology.benchmarks import trust_level44from aiatlas.registry import load, org_by_hf, org_ref, organizations45from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext46from aiatlas.sdk.extract.dates import parse_datetime47from aiatlas.sdk.facts import EntityRef, Facts, Target48from aiatlas.sdk.fetch import FetchResult495051def benchmark_ref(key: str) -> EntityRef | None:52 """Registry benchmark → EntityRef. `slug_hint` is the registry key; when an *organisation* already owns that slug (LiveBench the53 org vs `livebench` the benchmark) the resolver appends a collision suffix (`livebench-2` in production) — the identity is the54 `registry_benchmark` identifier, never the slug."""55 b = next((b for b in load("benchmarks") if b["key"] == key), None)56 if not b:57 return None58 return EntityRef(entity_type="benchmark", name=b["name"], identifiers={"registry_benchmark": key}, aliases=list(b.get("aliases", [])), slug_hint=key)596061def _slug(s: str) -> str:62 return re.sub(r"[^a-z0-9]+", "", s.lower())636465def org_from_name(facts: Facts, *names: str | None) -> EntityRef | None:66 """Registry organization by hf_org / key / name / alias (punctuation-insensitive: 'Z AI' == 'Z.ai'); models map to the lab, not the holding."""67 for n in names:68 if not n:69 continue70 low = n.strip().lower()71 known = org_by_hf(low)72 key = known["key"] if known else None73 if not key:74 wanted = _slug(low)75 for k, o in organizations().items():76 if wanted in {_slug(k), _slug(o["name"]), *(_slug(a) for a in o.get("aliases", []))}:77 key = k78 break79 if key:80 if key == "meta" and "meta-ai" in organizations():81 key = "meta-ai"82 return org_ref_in(facts, key)83 return None848586def _dt(v: Any) -> datetime | None:87 if isinstance(v, datetime):88 return v if v.tzinfo else v.replace(tzinfo=UTC)89 if isinstance(v, date):90 return datetime(v.year, v.month, v.day, tzinfo=UTC)91 return parse_datetime(v) if isinstance(v, str) else None929394def _clean(cfg: dict[str, Any]) -> dict[str, Any]:95 return {k: v for k, v in cfg.items() if v not in (None, "", [], {})}969798class _Leaderboard(BaseConnector):99 version = "2"100 parser_version = "2"101 interval_seconds = 12 * 3600102 min_interval_seconds = 6 * 3600103 max_interval_seconds = 3 * 86400104 tier = 2105 priority = 2106 concurrency = 1107108109# ================================================================================================ Aider polyglot110AIDER_SHORTCUTS = {"r1", "sonnet", "opus", "haiku", "4o", "4-turbo", "35turbo", "3", "flash", "deepseek", "gemini", "gemini-exp", "gemini-2.5-pro"}111112113class AiderLeaderboardConnector(_Leaderboard):114 name = "aider_leaderboard"115 label = "Aider polyglot coding leaderboard"116 description = "polyglot_leaderboard.yml (raw GitHub file of the aider repo): pass rate and well-formed rate per run, edit format, API id."117 source_key = "aider.chat"118 rate_per_min = 10119 expected_min_records = 30120 URL = "https://raw.githubusercontent.com/Aider-AI/aider/main/aider/website/_data/polyglot_leaderboard.yml"121 PAGE = "https://aider.chat/docs/leaderboards/"122123 async def discover(self, ctx: RunContext) -> list[Target]:124 return [Target(url=self.URL, doc_type="leaderboard", key="aider", min_bytes=1000, priority=1, meta={"content_type": "text/yaml"})]125126 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:127 facts = Facts()128 bench = benchmark_ref("aider-polyglot")129 well_formed = benchmark_ref("aider-polyglot-well-formed")130 if bench is None:131 return facts132 facts.entities.append(bench)133 if well_formed is not None:134 facts.entities.append(well_formed)135 try:136 rows = yaml.safe_load(res.text) or []137 except yaml.YAMLError:138 return facts139 trust = trust_level(self.source_key)140 for row in rows:141 if not isinstance(row, dict) or not row.get("model") or row.get("pass_rate_2") is None:142 continue143 label = str(row["model"]).strip()144 if re.search(r"\s\+\s", label) or row.get("edit_format") == "architect":145 continue # "o3 (high) + gpt-4.1": a two-model architect/editor system, not one model146 m = re.search(r"--model\s+(\S+)", str(row.get("command") or ""))147 api_id = m.group(1) if m else None148 trusted = bool(api_id) and api_id not in AIDER_SHORTCUTS149 # rolling ids (`deepseek/deepseek-chat`, `codestral-latest`) name whichever snapshot served that day: identifiers only for150 # pinned snapshots and namespaced (OpenRouter / hub) ids, aliases otherwise151 ident = model_identity(api_id, trusted=trusted) if trusted else None152 use_ids = bool(ident) and (ident.pinned or ident.provider_prefix == "openrouter" or "hf_repo" in ident.identifiers) # type: ignore[union-attr]153 ref, effort = model_ref_from_api_id(facts, api_id if trusted else None, name=label, trusted=use_ids, identity_confidence="medium")154 evaluated = _dt(row.get("date"))155 config = _clean({"edit_format": row.get("edit_format"), "api_model_id": api_id, "date": evaluated.date().isoformat() if evaluated else None,156 "reasoning_effort": row.get("reasoning_effort"), **effort})157 run_group = config.get("date")158 facts.result(model=ref, benchmark=bench, score=float(row["pass_rate_2"]), metric="pass_rate_2", unit="%", config=config, evaluated_at=evaluated,159 source_url=self.PAGE, trust_level=trust, variant="polyglot", run_group=run_group)160 if well_formed is not None and row.get("percent_cases_well_formed") is not None:161 facts.result(model=ref, benchmark=well_formed, score=float(row["percent_cases_well_formed"]), metric="percent_cases_well_formed", unit="%",162 config={**config, "variant": "well-formed"}, evaluated_at=evaluated, source_url=self.PAGE, trust_level=trust, variant="well-formed",163 run_group=run_group)164 facts.document_entity = bench165 facts.document_title = "Aider polyglot leaderboard data"166 return facts167168169# ================================================================================================ SWE-bench170BOARD_KEYS = {"verified": "swe-bench-verified", "lite": "swe-bench-lite", "test": "swe-bench-full", "full": "swe-bench-full",171 "multimodal": "swe-bench-multimodal", "multilingual": "swe-bench-multilingual", "pro": "swe-bench-pro"}172NOT_A_MODEL = {"multiple", "undisclosed", "unknown", "n/a", "-", "—", "various", "proprietary", "custom", "internal"}173174175class SweBenchLeaderboardConnector(_Leaderboard):176 name = "swebench_leaderboard"177 label = "SWE-bench leaderboards (Verified, Lite, Full, Multimodal, Multilingual)"178 description = "Embedded leaderboard JSON of swebench.com: % resolved per (system, model) submission with dates; trust follows the SWE-bench check mark."179 source_key = "swebench.com"180 rate_per_min = 10181 expected_min_records = 50182 URL = "https://www.swebench.com/"183184 async def discover(self, ctx: RunContext) -> list[Target]:185 return [Target(url=self.URL, doc_type="leaderboard", key="swebench", min_bytes=20000, priority=1)]186187 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:188 facts = Facts()189 html = parsed.html190 if not html:191 return facts192 data = html.embedded_json.get("leaderboard-data")193 if not isinstance(data, list):194 return facts195 for board in data:196 key = BOARD_KEYS.get(str(board.get("name") or "").lower())197 bench = benchmark_ref(key) if key else None198 if bench is None:199 continue200 facts.entities.append(bench)201 for r in board.get("results") or []:202 self._row(facts, bench, board.get("name"), r)203 facts.document_title = html.title or "SWE-bench leaderboards"204 return facts205206 def _row(self, facts: Facts, bench: EntityRef, board: str, r: dict[str, Any]) -> None:207 model_name = (r.get("model_display") or "").strip()208 resolved = r.get("resolved")209 if not model_name or not isinstance(resolved, (int, float)):210 return211 if re.search(r"\s(&|\+|and)\s", model_name): # "GPT-4o & Claude 3 Opus": a multi-model system, not one model212 return213 if model_name.lower() in NOT_A_MODEL: # "Multiple", "Undisclosed": the submission does not name its model214 return215 tags = [t for t in r.get("tags") or [] if isinstance(t, str)]216 model_tags = [t.split(":", 1)[1].strip() for t in tags if t.lower().startswith("model:")]217 model_tag = model_tags[0] if model_tags else None218 org = org_from_name(facts, r.get("model_org"))219 # the "Model:" tag is free text ("claude-4-5-opus", "gpt-5-2"): aliases and organisation only, never an identifier220 ref, effort = model_ref_from_api_id(facts, model_tag, name=model_name, trusted=False, identity_confidence="medium", organization=org)221 evaluated = _dt(r.get("date"))222 config = _clean({"board": board, "system": r.get("agent") or r.get("name"), "system_org": r.get("agent_org"), "open_source_system": r.get("os_system"),223 "checked_by_swebench": r.get("checked"), "submission": r.get("folder"), "date": evaluated.date().isoformat() if evaluated else None,224 "model_tag": model_tag, "reasoning_effort": r.get("reasoning_effort") or None, **effort})225 facts.result(model=ref, benchmark=bench, score=float(resolved), metric="resolved", unit="%", config=config, evaluated_at=evaluated, source_url=self.URL,226 trust_level=trust_level(self.source_key, config), variant=board, run_group=config.get("date"))227228229# ================================================================================================ LiveBench230RELEASES = re.compile(r'\[("20\d\d-\d\d-\d\d"(?:,"20\d\d-\d\d-\d\d")*)\]')231MODEL_META = re.compile(r'"([\w.:/-]+)":\{((?:[^{}]|\{[^{}]*\})*?displayName:"(?:[^{}]|\{[^{}]*\})*)\}')232META_FIELD = re.compile(r'(url|huggingface|organization|displayName):"([^"]*)"')233CATEGORY_KEYS = {"Reasoning": "livebench-reasoning", "Coding": "livebench-coding", "Agentic Coding": "livebench-agentic-coding", "Mathematics": "livebench-mathematics",234 "Data Analysis": "livebench-data-analysis", "Language": "livebench-language", "IF": "livebench-if", "Instruction Following": "livebench-if"}235236237class LiveBenchLeaderboardConnector(_Leaderboard):238 name = "livebench_leaderboard"239 label = "LiveBench leaderboard"240 description = "Latest LiveBench release table (CSV + categories JSON discovered from the site bundle): overall mean and one benchmark per category."241 source_key = "livebench.ai"242 rate_per_min = 10243 expected_min_records = 20244 BASE = "https://livebench.ai/"245246 async def discover(self, ctx: RunContext) -> list[Target]:247 return [Target(url=self.BASE, doc_type="listing", key="livebench_index", min_bytes=200, priority=1)]248249 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:250 facts = Facts()251 key = target.key or ""252 if key == "livebench_index" and parsed.html:253 script = next((s.attributes.get("src") for s in parsed.html.css("script[src]") if "main." in (s.attributes.get("src") or "")), None)254 if script:255 facts.follow(self.BASE + script.lstrip("./"), doc_type="script", key="livebench_js", min_bytes=10000, priority=1, meta={"content_type": "text/javascript"})256 elif key == "livebench_js":257 js = res.text258 dates = sorted({d for m in RELEASES.finditer(js) for d in json.loads("[" + m.group(1) + "]")})259 if not dates:260 ctx.log.warning("livebench bundle: no release list found")261 return facts262 latest = self.config.get("release") or dates[-1]263 models = {}264 for m in MODEL_META.finditer(js):265 fields = dict(META_FIELD.findall(m.group(2)))266 models[m.group(1)] = {"url": fields.get("url"), "huggingface": fields.get("huggingface"), "organization": fields.get("organization"),267 "display_name": fields.get("displayName")}268 tag = latest.replace("-", "_")269 facts.follow(f"{self.BASE}categories_{tag}.json", doc_type="data", key="livebench_categories", min_bytes=20, priority=1,270 meta={"release": latest, "models": models, "content_type": "application/json"})271 elif key == "livebench_categories":272 cats = parsed.json if parsed.kind == "json" else None273 release = target.meta.get("release")274 if isinstance(cats, dict) and release:275 facts.follow(f"{self.BASE}table_{release.replace('-', '_')}.csv", doc_type="leaderboard", key="livebench_table", min_bytes=200, priority=1,276 meta={"release": release, "categories": cats, "models": target.meta.get("models") or {}, "content_type": "text/csv"})277 elif key == "livebench_table":278 self._table(facts, target, res.text)279 return facts280281 def _table(self, facts: Facts, target: Target, text: str) -> None:282 bench = benchmark_ref("livebench")283 if bench is None:284 return285 facts.entities.append(bench)286 release = target.meta.get("release")287 cats: dict[str, list[str]] = target.meta.get("categories") or {}288 meta: dict[str, dict[str, Any]] = target.meta.get("models") or {}289 evaluated = _dt(release)290 trust = trust_level(self.source_key)291 cat_benches: dict[str, EntityRef] = {}292 for cat in cats:293 ckey = CATEGORY_KEYS.get(cat) or f"livebench-{re.sub(r'[^a-z0-9]+', '-', cat.lower()).strip('-')}"294 cref = benchmark_ref(ckey)295 if cref is not None:296 cat_benches[cat] = cref297 facts.entities.append(cref)298 for row in csv.DictReader(io.StringIO(text)):299 mid = (row.get("model") or "").strip()300 if not mid:301 continue302 info = meta.get(mid) or {}303 base_id, effort = strip_effort(mid)304 display, label_effort = split_effort_label(info.get("display_name") or base_id) # "GPT-5.6 Sol xHigh Effort" → base + setting305 effort = {**effort, **label_effort}306 if effort and info.get("display_name"):307 display = strip_effort_words(display, only_if=True)308 # no `hf_repo` identifier here: several LiveBench ids (effort variants, MLX conversions) share one display name and would309 # glue unrelated hub repositories onto a single entity; the link is kept as a claim on the leaderboard row's model310 org = org_from_name(facts, info.get("organization"))311 if org is None:312 _scheme, org_key = org_key_for_bare_id(base_id)313 org = org_ref_in(facts, org_key)314 ref = next((e for e in facts.entities if e.entity_type == "model" and e.name.lower() == display.lower()), None)315 if ref is None:316 ref = facts.entity("model", display[:200], organization=org, aliases=[a for a in dict.fromkeys([base_id, mid]) if a != display],317 identity_confidence="medium", family=family_ref(display, org))318 elif mid not in ref.aliases:319 ref.aliases.append(mid)320 hf = info.get("huggingface") or ""321 m = re.search(r"huggingface\.co/([\w.-]+/[\w.-]+)", hf)322 if m:323 facts.claim(ref, "livebench_hf_link", f"https://huggingface.co/{m.group(1)}")324 scores = {k: float(v) for k, v in row.items() if k and k != "model" and v not in (None, "", "-") and _is_number(v)}325 cat_avgs: dict[str, float] = {}326 for cat, tasks in cats.items():327 vals = [scores[t] for t in tasks if t in scores]328 if vals:329 cat_avgs[cat] = round(fmean(vals), 3)330 if not cat_avgs:331 continue332 base_cfg = _clean({"release": release, "livebench_model_id": mid, "aggregation": "mean of category averages; category = mean of its subtasks", **effort})333 facts.result(model=ref, benchmark=bench, score=round(fmean(cat_avgs.values()), 3), metric="global_average", unit="%", config=base_cfg,334 evaluated_at=evaluated, source_url=self.BASE, trust_level=trust, variant="global", run_group=release)335 for cat, avg in cat_avgs.items():336 cref = cat_benches.get(cat)337 if cref is None:338 continue339 facts.result(model=ref, benchmark=cref, score=avg, metric="average score", unit="%",340 config=_clean({"release": release, "livebench_model_id": mid, "variant": cat, "subtasks": cats[cat], **effort}),341 evaluated_at=evaluated, source_url=self.BASE, trust_level=trust, variant=cat, run_group=release)342 facts.document_entity = bench343 facts.document_title = f"LiveBench {release}"344345346def _is_number(v: str) -> bool:347 try:348 float(v)349 return True350 except ValueError:351 return False352353354# ================================================================================================ Artificial Analysis355# RSC field → (registry benchmark key, metric, variant label). GPQA on AA is the Diamond subset; τ²-Bench is the Telecom domain.356AA_EVALS = {"gpqa": ("gpqa-diamond", "accuracy", "Diamond"), "hle": ("humanitys-last-exam", "accuracy", None), "aime25": ("aime-2025", "accuracy", None),357 "livecodebench": ("livecodebench", "pass@1", None), "scicode": ("scicode", "accuracy", None), "ifbench": ("ifbench", "accuracy", None),358 "mmmuPro": ("mmmu-pro", "accuracy", None), "tau2": ("tau2-bench", "pass^1", "Telecom"), "terminalbenchV40": ("terminal-bench", "accuracy", "v4.0"),359 "terminalbenchV21": ("terminal-bench", "accuracy", "v2.1"), "terminalbenchHard": ("terminal-bench", "accuracy", "hard")}360PUSH = re.compile(r'self\.__next_f\.push\(\[1,"(.*?)"\]\)\s*</script>', re.DOTALL)361# model-level facts AA copies from the labs (same for every effort variant of a model) → written once per base model as `aa_*`362AA_MODEL_CLAIMS = ("aa_release_date", "aa_openness", "aa_context_window", "aa_deprecated")363364365class ArtificialAnalysisConnector(_Leaderboard):366 name = "artificial_analysis"367 label = "Artificial Analysis — Intelligence Index and component evaluations"368 description = "Model leaderboard page (RSC payload): Intelligence Index with version and component evaluations; effort variants fold into their base model."369 source_key = "artificialanalysis.ai"370 rate_per_min = 6371 expected_min_records = 100372 URL = "https://artificialanalysis.ai/leaderboards/models"373374 async def discover(self, ctx: RunContext) -> list[Target]:375 return [Target(url=self.URL, doc_type="leaderboard", key="aa_models", min_bytes=50000, priority=1)]376377 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:378 facts = Facts()379 models, version = parse_rsc_models(res.text)380 if not models:381 ctx.log.warning("artificial analysis: no model objects in the RSC payload")382 return facts383 index = benchmark_ref("artificial-analysis-intelligence-index")384 assert index385 facts.entities.append(index)386 benches = {k: benchmark_ref(k) for k, *_ in AA_EVALS.values()}387 trust = trust_level(self.source_key)388 written: set[tuple[int, str]] = set()389390 def once(ref: EntityRef, prop: str, value: Any, **kw: Any) -> None:391 """One claim per (base model, property): several effort variants of a model repeat the same model-level facts."""392 if value in (None, "", []) or (id(ref), prop) in written:393 return394 written.add((id(ref), prop))395 facts.claim(ref, prop, value, **kw)396397 for slug, m in models.items():398 ref, effort = self._model(facts, slug, m, models)399 # AA copies release date / openness / context / deprecation from the labs: recorded under `aa_*` so that a second-hand400 # tier-2 source never supersedes another tier-2 source (hub/OpenRouter) every run; the results are AA's own data.401 if m.get("releaseDate"):402 once(ref, "aa_release_date", str(m["releaseDate"])[:10])403 if isinstance(m.get("isOpenWeights"), bool):404 once(ref, "aa_openness", "open-weights" if m["isOpenWeights"] else "proprietary")405 if m.get("deprecated") is True and not effort:406 once(ref, "aa_deprecated", True)407 if isinstance(m.get("contextWindowTokens"), int) and m["contextWindowTokens"] > 0:408 once(ref, "aa_context_window", m["contextWindowTokens"], unit="tokens")409 if not effort:410 if isinstance(m.get("isReasoning"), bool):411 once(ref, "reasoning", m["isReasoning"])412 if isinstance(m.get("medianOutputTokensPerSecond"), (int, float)):413 once(ref, "metric.aa_median_output_tokens_per_second", round(float(m["medianOutputTokensPerSecond"]), 1))414 # the reasoning switch is part of every result's configuration (a base row and its "non-reasoning" sibling differ by it)415 if isinstance(m.get("isReasoning"), bool) and "reasoning" not in effort and "reasoning_effort" not in effort:416 effort["reasoning"] = "on" if m["isReasoning"] else "off"417 ii = m.get("intelligenceIndex")418 if isinstance(ii, (int, float)):419 cfg = _clean({"version": version, "estimated": bool(m.get("intelligenceIndexIsEstimated")), "aa_slug": slug, **effort})420 facts.result(model=ref, benchmark=index, score=round(float(ii), 2), metric="index", unit="", config=cfg, source_url=self.URL, trust_level=trust,421 variant="index", run_group=version)422 for field, (bkey, metric, variant) in AA_EVALS.items():423 v = m.get(field)424 bench = benches.get(bkey)425 if bench is None or not isinstance(v, (int, float)):426 continue427 if bench not in facts.entities:428 facts.entities.append(bench)429 cfg = _clean({"evaluator": "Artificial Analysis", "index_version": version, "aa_slug": slug, "variant": variant, **effort})430 facts.result(model=ref, benchmark=bench, score=round(float(v) * 100, 2), metric=metric, unit="%", config=cfg, source_url=self.URL, trust_level=trust,431 variant=variant, run_group=version)432 facts.document_entity = index433 facts.document_title = f"Artificial Analysis models — Intelligence Index v{version}" if version else "Artificial Analysis models"434 return facts435436 def _model(self, facts: Facts, slug: str, m: dict[str, Any], models: dict[str, dict[str, Any]]) -> tuple[EntityRef, dict[str, str]]:437 """Base-model EntityRef + effort configuration for one AA row. AA's own base label is `shortName` without its parenthetical438 ("Claude Opus 5 (xhigh)" → "Claude Opus 5"); the slug carries the suffix (`claude-opus-5-xhigh`). The `artificial_analysis`439 identifier is attached only when AA lists the base slug itself; otherwise the base slug is an alias."""440 short = (m.get("shortName") or m.get("name") or slug).strip()441 label, label_effort = split_effort_label(short)442 base_slug, slug_effort = strip_effort(slug)443 effort = {**slug_effort, **label_effort}444 if slug_effort and not label_effort:445 label = strip_effort_words(label, only_if=True)446 variant = bool(effort) and base_slug != slug447 creator = m.get("creator") or {}448 creator_name = creator.get("name") or m.get("modelCreatorName")449 org = org_from_name(facts, creator.get("slug"), creator_name)450 if org is None and org_key_for_bare_id(base_slug)[1] is None and creator_name:451 # creator unknown to the registry and no first-party pattern: AA's own creator label becomes a company with AA's identifier452 existing = next((e for e in facts.entities if e.identifiers.get("artificial_analysis_creator") == (creator.get("slug") or _slug(creator_name))), None)453 org = existing or facts.entity("company", creator_name, identifiers={"artificial_analysis_creator": creator.get("slug") or _slug(creator_name)})454 identifiers: dict[str, str] = {}455 aliases: list[str] = []456 if not variant:457 identifiers["artificial_analysis"] = slug458 elif base_slug in models:459 identifiers["artificial_analysis"] = base_slug460 else:461 aliases.append(base_slug)462 aliases += [a for a in (short if not effort else None, m.get("shortName") if not effort else None) if a]463 ref, _ = model_ref_from_api_id(facts, base_slug, name=label or base_slug, trusted=False, identity_confidence="medium", organization=org,464 extra_aliases=aliases)465 for k, v in identifiers.items():466 ref.identifiers.setdefault(k, v)467 return ref, effort468469470def parse_rsc_models(html: str) -> tuple[dict[str, dict[str, Any]], str | None]:471 """Concatenate the Next.js flight chunks, parse each `id:json` line and collect every object with `slug` + `intelligenceIndex`."""472 chunks = []473 for p in PUSH.findall(html):474 try:475 chunks.append(json.loads('"' + p + '"'))476 except json.JSONDecodeError:477 continue478 blob = "".join(chunks)479 version_m = re.search(r"Intelligence Index v(\d+(?:\.\d+)?)", blob)480 models: dict[str, dict[str, Any]] = {}481 for line in blob.split("\n"):482 _, sep, payload = line.partition(":")483 if not sep or "intelligenceIndex" not in payload:484 continue485 try:486 data = json.loads(payload)487 except json.JSONDecodeError:488 continue489 stack = [data]490 while stack:491 o = stack.pop()492 if isinstance(o, dict):493 if "intelligenceIndex" in o and isinstance(o.get("slug"), str):494 models.setdefault(o["slug"], o)495 else:496 stack.extend(o.values())497 elif isinstance(o, list):498 stack.extend(o)499 return models, (version_m.group(1) if version_m else None)500501502_ = org_ref # kept importable for tests/back-compat503504CONNECTORS = [AiderLeaderboardConnector, SweBenchLeaderboardConnector, LiveBenchLeaderboardConnector, ArtificialAnalysisConnector]505