HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Mistral AI — docs models overview (Next.js, server-rendered cards + deprecation table), model pages (embedded pricing JSON), news RSS.23Sources (tier 1):4 * models overview → docs.mistral.ai/models (the old /getting-started/models/models_overview/ URL redirects here):5 cards `a[href^=/models/]` (name, weights-license badge, description, version) and the6 "Deprecated & retired models" table (name, version, API id, deprecation / retirement dates, alternative)7 * model pages → docs.mistral.ai/models/<slug> : context window, "Released as open weights under a … license", features,8 RSC payload `"pricing":{input:[{price}], output:[{price}]}` in USD per 1M tokens (EUR kept in features), `isRetired`9 * news → mistral.ai/news/rss (RSS feed linked from the news listing) → ANNOUNCEMENT events1011mistral.ai/pricing is server-rendered without any per-model price (plans only; model prices load client-side), so it is not crawled:12provider prices come from the docs model pages instead.13"""14from __future__ import annotations1516import re17from datetime import datetime18from typing import Any1920from selectolax.parser import HTMLParser2122from aiatlas.connectors._identity import family_ref23from aiatlas.registry import org_ref, provider_ref24from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext25from aiatlas.sdk.extract.dates import parse_datetime26from aiatlas.sdk.extract.html import node_text27from aiatlas.sdk.facts import EntityRef, Facts, Target28from aiatlas.sdk.fetch import FetchResult2930from ._common import (31 announcement_events,32 claim_api_aliases,33 claim_license,34 claim_status,35 json_after,36 model_ref,37 next_flight_payload,38 normalize_capabilities,39 slug_of,40 tokens,41)4243DOCS = "https://docs.mistral.ai"44NEWS_RSS = "https://mistral.ai/news/rss"45PROVIDER_KEY = "mistral"46SLUG_SCHEME = "mistral_docs_slug"47ID_SCHEME = "mistral_model_id"48MAX_MODEL_PAGES = 504950API_ID = re.compile(r"^[a-z0-9][a-z0-9.\-]*$")51OPEN_LICENSES = re.compile(r"apache|mit\b|mrl|research licen[cs]e|open[- ]weight|cc-by|gpl|bsd", re.IGNORECASE)52LICENSE_SENTENCE = re.compile(r"(?:released|available|distributed)\s+(?:as\s+)?open[- ]weights?\s+under\s+(?:a\s+|an\s+|the\s+)?(.+?)\s+licen[cs]e", re.IGNORECASE)53US_DATE = re.compile(r"\b(\d{1,2})/(\d{1,2})/(\d{4})\b")545556class MistralConnector(BaseConnector):57 name = "mistral"58 label = "Mistral AI — models, model pages (pricing), news"59 description = "Mistral docs models overview (cards + deprecation table), model pages with embedded pricing, and the Mistral news RSS."60 source_key = "docs.mistral.ai"61 version = "1"62 parser_version = "1"63 interval_seconds = 360064 min_interval_seconds = 180065 rate_per_min = 1266 tier = 167 priority = 068 expected_min_records = 4069 concurrency = 27071 async def discover(self, ctx: RunContext) -> list[Target]:72 return [73 Target(url=f"{DOCS}/models", doc_type="model_docs", key="models", min_bytes=20000),74 Target(url=NEWS_RSS, doc_type="feed", key="news", min_bytes=1000),75 ]7677 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:78 facts = Facts()79 org = org_ref("mistral")80 facts.entities.append(org)81 key = target.key or ""82 if key == "models" and parsed.html:83 self._models(facts, org, parsed, res)84 elif key == "news" and parsed.kind == "feed":85 announcement_events(facts, org, parsed.feed_items, source_name="mistral.ai/news", max_follow=15)86 facts.document_title, facts.document_entity = "Mistral AI news", org87 elif target.doc_type == "model_page" and parsed.html:88 self._model_page(facts, org, target, res, parsed)89 return facts9091 # ------------------------------------------------------------------------------------------ models overview92 def _models(self, facts: Facts, org: EntityRef, parsed: Parsed, res: FetchResult) -> None:93 observed: datetime = res.fetched_at94 html = parsed.html95 assert html96 facts.document_title, facts.document_entity = "Mistral models overview", org97 # the SDK's parse_html strips <button> (and nav/footer/form) from the tree it keeps; the licence badge lives inside a button,98 # so the cards are read from a fresh parse of the raw document99 tree = HTMLParser(res.content)100 active: list[str] = []101 names: dict[str, str] = {} # display name (lower) -> slug of the first model carrying it (Mistral reuses names across versions)102 for card in tree.css('a[href^="/models/"]'):103 slug = slug_of(card.attributes.get("href") or "")104 h3 = card.css_first("h3")105 if not h3 or not slug:106 continue107 version = next((m.group(1) for m in (re.fullmatch(r"v\s*(\d[\w.\-]*)", node_text(div)) for div in card.css("div")) if m), None)108 name = _unique_name(names, node_text(h3), slug, version)109 desc = card.css_first("p")110 desc_text = node_text(desc) if desc else ""111 developer = _third_party(name, desc_text)112 ref = _model(facts, developer or org, name, slug, third_party=developer is not None)113 badge = card.css_first("span[data-slot='badge']")114 if badge:115 _license_claims(facts, ref, node_text(badge))116 facts.claim(ref, "description", desc_text or None)117 facts.claim(ref, "version", version)118 if slug not in active:119 active.append(slug)120 deprecated: list[str] = []121 for tr in tree.css("table tr"):122 tds = tr.css("td")123 if len(tds) != 5:124 continue125 link = tds[0].css_first("a[href]")126 name = re.sub(r"\s*[↗→]\s*$", "", node_text(tds[0])).strip()127 slug = slug_of(link.attributes.get("href") or "") if link else ""128 api_id = node_text(tds[2]).strip()129 if not name:130 continue131 name = _unique_name(names, name, slug, node_text(tds[1]) or None)132 ref = _model(facts, org, name, slug or None, api_id if API_ID.match(api_id) else None)133 facts.claim(ref, "version", node_text(tds[1]) or None)134 dates = [_us_date(m) for m in US_DATE.finditer(node_text(tds[3]))]135 dep, ret = (dates + [None, None])[:2]136 facts.claim(ref, "deprecation_date", dep)137 facts.claim(ref, "retirement_date", ret)138 claim_status(facts, ref, "retired" if ret and ret < observed.date().isoformat() else "deprecated")139 alt_link = tds[4].css_first("a[href]")140 alt_name = node_text(tds[4]).strip()141 if alt_name and alt_name != name:142 alt = _model(facts, org, alt_name, slug_of(alt_link.attributes.get("href") or "") if alt_link else None)143 facts.relate(ref, "superseded_by", alt, attributes={"deprecated": dep, "retired": ret})144 if slug:145 deprecated.append(slug)146 for slug in [s for s in active if s not in deprecated][:MAX_MODEL_PAGES]:147 facts.follow(f"{DOCS}/models/{slug}", doc_type="model_page", key=f"model:{slug}", meta={"slug": slug}, min_bytes=20000, priority=1)148149 # ------------------------------------------------------------------------------------------ model page150 def _model_page(self, facts: Facts, org: EntityRef, target: Target, res: FetchResult, parsed: Parsed) -> None:151 html = parsed.html152 assert html153 slug = target.meta.get("slug") or slug_of(res.final_url or res.url)154 name = next((t for lvl, t in html.headings if lvl == 1), None)155 if not name or not slug:156 return157 ref = _model(facts, org, name, slug)158 facts.document_title, facts.document_entity = name, ref159 facts.claim(ref, "official_url", f"{DOCS}/models/{slug}")160 text = html.text161 m = re.search(r"\bContext\s+(\d+(?:\.\d+)?\s*[kKmM]\b|\d{4,})", text)162 if m:163 facts.claim(ref, "context_length", tokens(m.group(1)), unit="tokens")164 intro = next((node_text(p) for p in html.css("p") if LICENSE_SENTENCE.search(node_text(p))), None)165 m = LICENSE_SENTENCE.search(intro or "")166 if m:167 facts.claim(ref, "description", intro.split(" Released")[0].strip() if intro and " Released" in intro else None)168 claim_license(facts, ref, m.group(1).strip(), weights_available=True)169 features_section = html.css_first("h3#features, h3[id*='features']")170 feats = [node_text(n) for n in html.css("[class*='LinkItem_link'] span, [class*='LinkItem_link'] p")] if features_section is None else []171 blob = " | ".join(feats) if feats else text172 caps: list[str] = []173 for label, prop in (("Function Calling", "tool_calling"), ("Structured Outputs", "structured_output"), ("Vision", "vision"), ("Reasoning", "reasoning")):174 if re.search(rf"\b{label}\b", blob):175 facts.claim(ref, prop, True)176 caps.append(label)177 if caps:178 facts.claim(ref, "capabilities", normalize_capabilities(caps))179 payload = next_flight_payload(res.content)180 if payload:181 pricing = json_after(payload, '"pricing":')182 if isinstance(pricing, dict):183 provider = provider_ref(PROVIDER_KEY)184 facts.entities.append(provider)185 p_in = _first_price(pricing.get("input"))186 p_out = _first_price(pricing.get("output"))187 if p_in is not None or p_out is not None:188 feats_d: dict[str, Any] = {"free": bool(pricing.get("free"))}189 eur_in, eur_out = _first_price(pricing.get("input"), key="priceEur"), _first_price(pricing.get("output"), key="priceEur")190 if eur_in is not None:191 feats_d["eur_input_per_mtok"] = eur_in192 if eur_out is not None:193 feats_d["eur_output_per_mtok"] = eur_out194 facts.price(model=ref, provider=provider, input_per_mtok=p_in, output_per_mtok=p_out, features=feats_d, meta={"from": "model page"})195 if re.search(r'"isRetired":true', payload):196 claim_status(facts, ref, "retired")197 names = json_after(payload, '"names":')198 if isinstance(names, list):199 latest = [n for n in names if isinstance(n, str) and n.endswith("-latest")]200 claim_api_aliases(facts, ref, latest)201202203# ---------------------------------------------------------------------------------------------- helpers204def _model(facts: Facts, org: EntityRef, name: str, slug: str | None, api_id: str | None = None, *, third_party: bool = False) -> EntityRef:205 for e in facts.entities:206 if e.entity_type != "model":207 continue208 if slug and e.identifiers.get(SLUG_SCHEME) == slug:209 if api_id and ID_SCHEME not in e.identifiers:210 e.identifiers[ID_SCHEME] = api_id211 facts.claim(e, "api_model_id", api_id)212 return e213 if not slug and e.name.lower() == name.lower():214 return e215 ids: dict[str, str] = {}216 if slug:217 ids[SLUG_SCHEME] = slug218 if api_id:219 ids[ID_SCHEME] = api_id220 ref = facts.entity("model", name, identifiers=ids, organization=org, aliases=[api_id] if api_id else [], family=family_ref(name, org),221 identity_confidence="high" if ids else "medium")222 if org not in facts.entities:223 facts.entities.append(org)224 facts.relate(org, "develops", ref)225 if third_party:226 facts.relate(ref, "available_through", provider_ref(PROVIDER_KEY))227 facts.claim(ref, "api_model_id", api_id)228 if slug:229 facts.claim(ref, "official_url", f"{DOCS}/models/{slug}")230 return ref231232233def _unique_name(names: dict[str, str], name: str, slug: str, version: str | None) -> str:234 """'Codestral' exists as 24.05, 25.01 and 25.08: the first keeps the bare name, later versions get it appended so that235 alias resolution never folds distinct versions into one entity."""236 key = name.lower()237 if slug and names.setdefault(key, slug) != slug:238 return f"{name} {version}" if version and not name.endswith(version) else f"{name} ({slug})"239 return name240241242THIRD_PARTY_ORGS = {"z.ai": "zhipu", "glm": "zhipu"}243244245def _third_party(name: str, description: str) -> EntityRef | None:246 """Models hosted by Mistral but developed elsewhere ("A third-party open source text model from Z.ai") → their own registry org."""247 if "third-party" not in description.lower() and "third party" not in description.lower():248 return None249 low = name.lower()250 for prefix, key in THIRD_PARTY_ORGS.items():251 if low.startswith(prefix):252 return org_ref(key)253 return None254255256def _license_claims(facts: Facts, ref: EntityRef, badge: str) -> None:257 """Weights-licence badge ("Apache 2.0", "MRL", "Modified MIT", "Proprietary") → canonical `license` (+ `license_raw`) and the openness258 category derived from the ontology (MRL → restricted-weights, Apache 2.0 → open-weights)."""259 badge = badge.strip()260 if not badge:261 return262 low = badge.lower()263 if OPEN_LICENSES.search(badge):264 claim_license(facts, ref, badge, weights_available=True)265 elif "proprietary" in low or "commercial" in low or "api only" in low or "closed" in low:266 facts.claim(ref, "openness", "proprietary")267 facts.claim(ref, "weights_available", False)268 if "licen" in low:269 claim_license(facts, ref, badge)270271272def _us_date(m: re.Match[str]) -> str | None:273 dt = parse_datetime(f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}")274 return dt.date().isoformat() if dt else None275276277def _first_price(entries: Any, *, key: str = "price") -> float | None:278 if not isinstance(entries, list):279 return None280 for e in entries:281 if isinstance(e, dict) and isinstance(e.get(key), (int, float)) and "token" in str(e.get("denominator", "/M Tokens")).lower():282 return float(e[key])283 return None284285286_ = model_ref # shared helper kept importable for symmetry with the other lab connectors287288CONNECTORS = [MistralConnector]289