HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Cohere — docs models overview (served as Markdown by appending `.md`) + blog listing (Next.js RSC payload).23Sources (tier 1):4 * models → docs.cohere.com/docs/models.md : one table per family (Command, North, Embed, Rerank, Parse, Audio, Aya…) with model id,5 status, description, modalities, context length, max output, endpoints; platform tables (Bedrock / SageMaker / Azure / OCI ids)6 * blog → cohere.com/blog : post objects (title, slug, date, tags) embedded in the React Server Components payload → ANNOUNCEMENT events7"""8from __future__ import annotations910import re11from typing import Any1213from aiatlas.ontology.openness import normalize_openness14from aiatlas.ontology.taxonomy import normalize_modalities15from aiatlas.registry import org_ref, provider_ref16from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext17from aiatlas.sdk.extract.dates import parse_datetime18from aiatlas.sdk.extract.feeds import FeedItem19from aiatlas.sdk.extract.numbers import parse_active_params, parse_param_count20from aiatlas.sdk.facts import EntityRef, Facts, Target21from aiatlas.sdk.fetch import FetchResult2223from ._common import (24 announcement_events,25 claim_api_aliases,26 claim_status,27 clean_cell,28 model_ref,29 next_flight_payload,30 tokens,31)3233DOCS = "https://docs.cohere.com/docs/models"34BLOG = "https://cohere.com/blog"35PROVIDER_KEY = "cohere"36ID_SCHEME = "cohere_model_id"37API_ID = re.compile(r"^[a-z0-9][a-z0-9.\-]*$")38ALIAS_FOR = re.compile(r"^Alias for `?([a-z0-9.\-]+)`?", re.IGNORECASE)39PLATFORM_COLUMNS = {"amazon bedrock model id": "bedrock_model_id", "azure ai foundry": "foundry_model_id", "oracle oci generative ai service": "oci_model_id"}40NOT_AN_ID = re.compile(r"n/a|unique per deployment|coming soon", re.IGNORECASE)41MODALITIES = {"text": "text", "images": "image", "image": "image", "audio": "audio", "video": "video"}424344class CohereConnector(BaseConnector):45 name = "cohere"46 label = "Cohere — models overview, blog"47 description = "Cohere docs models overview (Markdown tables per family + platform ids) and the Cohere blog listing."48 source_key = "cohere.com"49 version = "1"50 parser_version = "1"51 interval_seconds = 720052 min_interval_seconds = 360053 rate_per_min = 1254 tier = 155 priority = 156 expected_min_records = 2557 concurrency = 25859 async def discover(self, ctx: RunContext) -> list[Target]:60 return [61 Target(url=f"{DOCS}.md", doc_type="model_docs", key="models", min_bytes=5000, meta={"content_type": "text/markdown"}),62 Target(url=BLOG, doc_type="listing", key="blog", min_bytes=20000),63 ]6465 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:66 facts = Facts()67 org = org_ref("cohere")68 facts.entities.append(org)69 key = target.key or ""70 if key == "models" and parsed.markdown:71 self._models(facts, org, parsed)72 elif key == "blog" and parsed.html:73 items = blog_items(res.content) or [FeedItem(id=h, url=h, title=t, summary=None, published_at=None, updated_at=None)74 for h, t in parsed.html.links_matching(r"^https://cohere\.com/blog/(?!tag/)[^/?#]+$") if len(t) > 12]75 announcement_events(facts, org, items, source_name="cohere.com/blog", max_follow=15)76 facts.document_title, facts.document_entity = "The Cohere Blog", org77 return facts7879 # ------------------------------------------------------------------------------------------ models overview80 def _models(self, facts: Facts, org: EntityRef, parsed: Parsed) -> None:81 md = parsed.markdown82 assert md83 facts.document_title, facts.document_entity = "Cohere models", org84 provider = provider_ref(PROVIDER_KEY)85 facts.entities.append(provider)86 families = _table_families(md.body)87 aliases: list[tuple[str, str, dict[str, str]]] = [] # (alias id, target id, row)88 for i, t in enumerate(md.tables):89 hs = [clean_cell(h).lower() for h in t["headers"]]90 if not hs or "model name" not in hs[0]:91 continue92 family = families[i] if i < len(families) else None93 is_platform = any(h in PLATFORM_COLUMNS or "sagemaker" in h for h in hs[1:])94 for r in t["rows"]:95 if len(r) < 2:96 continue97 row = {h: clean_cell(c) for h, c in zip(hs, r, strict=False)}98 api_id = row.get(hs[0], "")99 if not API_ID.match(api_id):100 continue101 if is_platform:102 # alias rows (command-r-plus → command-r-plus-04-2024) carry the platform ids of the model they point at103 target_id = next((t for a, t, _row in aliases if a == api_id), api_id)104 ref = _model(facts, org, target_id, family)105 if target_id != api_id and api_id not in ref.aliases:106 ref.aliases.append(api_id)107 for h, prop in PLATFORM_COLUMNS.items():108 v = row.get(h, "")109 if v and not NOT_AN_ID.search(v):110 facts.claim(ref, prop, v.strip("'’‘"))111 continue112 desc = row.get("description", "")113 m_alias = ALIAS_FOR.match(desc)114 if m_alias:115 aliases.append((api_id, m_alias.group(1), row))116 continue117 ref = _model(facts, org, api_id, family)118 self._row_claims(facts, ref, row, desc)119 facts.relate(provider, "available_through", ref) if False else None # (relation direction: model available_through provider)120 facts.relate(ref, "available_through", provider, attributes={"endpoints": _endpoints(row.get("endpoints", ""))})121 for alias, target_id, row in aliases:122 ref = next((e for e in facts.entities if e.entity_type == "model" and e.identifiers.get(ID_SCHEME) == target_id), None)123 if ref is None:124 ref = _model(facts, org, target_id, None)125 if alias not in ref.aliases:126 ref.aliases.append(alias)127 existing = next((c.value for c in facts.claims if c.entity is ref and c.property == "api_aliases"), [])128 facts.claims = [c for c in facts.claims if not (c.entity is ref and c.property in ("api_aliases", "api_alias"))]129 claim_api_aliases(facts, ref, [*existing, alias])130 status, dep = _status(row.get("status", ""))131 if status == "deprecated" and dep:132 facts.claim(ref, "alias_deprecation_date", dep)133134 def _row_claims(self, facts: Facts, ref: EntityRef, row: dict[str, str], desc: str) -> None:135 facts.claim(ref, "description", desc or None)136 status, dep = _status(row.get("status", ""))137 claim_status(facts, ref, status)138 facts.claim(ref, "deprecation_date", dep)139 mods_raw = row.get("modality") or row.get("modalities") or ""140 mods = normalize_modalities(_modalities(mods_raw))141 if mods:142 facts.claim(ref, "modalities", mods)143 facts.claim(ref, "modalities_input", mods)144 facts.claim(ref, "vision", "image" in mods)145 ctx = row.get("context length", "")146 if ctx:147 facts.claim(ref, "context_length", tokens(ctx), unit="tokens")148 out = row.get("maximum output tokens", "")149 if out:150 facts.claim(ref, "max_output_tokens", tokens(out), unit="tokens")151 dims = row.get("dimensions", "")152 if dims:153 nums = [int(x) for x in re.findall(r"\d{2,5}", dims)]154 facts.claim(ref, "embedding_dimensions", sorted(set(nums)) if nums else None)155 facts.claim(ref, "modalities_output", ["embedding"])156 facts.claim(ref, "endpoints", _endpoints(row.get("endpoints", "")) or None)157 low = desc.lower()158 if re.search(r"\bMoE\b|mixture[- ]of[- ]experts", desc, re.IGNORECASE):159 facts.claim(ref, "is_moe", True)160 if "open-weight" in low or "open weight" in low or "open source" in low or "open-source" in low:161 # the docs prose says "open"; the weights exist on the hub (Cohere Labs) — the category itself is derived from the licence elsewhere162 facts.claim(ref, "weights_available", True)163 facts.claim(ref, "openness", normalize_openness("open-weights"))164 facts.claim(ref, "openness_raw", "open-source" if "open source" in low or "open-source" in low else "open-weights")165 if re.search(r"\d+(?:\.\d+)?B\s+(total|parameter|params|instruct|model)", desc):166 facts.claim(ref, "parameter_count", parse_param_count(desc))167 active = parse_active_params(desc)168 facts.claim(ref, "active_parameter_count", active)169 if active or re.search(r"\bMoE\b|mixture of experts", desc, re.IGNORECASE):170 facts.claim(ref, "is_moe", True)171 if re.search(r"\breasoning model\b|able to 'think'|think before", low):172 facts.claim(ref, "reasoning", True)173 if "tool use" in low or "tool-use" in low or "function calling" in low:174 facts.claim(ref, "tool_calling", True)175176177# ---------------------------------------------------------------------------------------------- helpers178def _model(facts: Facts, org: EntityRef, api_id: str, family: str | None) -> EntityRef:179 for e in facts.entities:180 if e.entity_type == "model" and e.identifiers.get(ID_SCHEME) == api_id:181 return e182 ref = model_ref(facts, api_id, org, api_id=api_id, provider_key=PROVIDER_KEY, family=family)183 facts.claim(ref, "api_model_id", api_id)184 facts.claim(ref, "official_url", DOCS)185 return ref186187188def _status(cell: str) -> tuple[str | None, str | None]:189 c = cell.strip()190 if not c:191 return None, None192 low = c.lower()193 if low == "live":194 return "active", None195 if low.startswith("deprecated"):196 dt = parse_datetime(re.sub(r"(?i)^deprecated\s*", "", c).replace("Sept", "Sep")) if len(c) > 10 else None197 return "deprecated", dt.date().isoformat() if dt else None198 if "retired" in low or "removed" in low:199 return "retired", None200 if "preview" in low or "beta" in low:201 return "preview", None202 return None, None203204205def _modalities(cell: str) -> list[str]:206 out: list[str] = []207 for tok in re.split(r",|/|\band\b|\(", cell.lower()):208 tok = tok.strip().strip(")").strip()209 for word, mod in MODALITIES.items():210 if tok.startswith(word) and mod not in out:211 out.append(mod)212 return out213214215def _endpoints(cell: str) -> list[str]:216 return [e.strip() for e in re.split(r",", clean_cell(cell)) if e.strip()]217218219def _table_families(body: str) -> list[str | None]:220 """The h2 heading (model family) preceding each Markdown table, in table order."""221 out: list[str | None] = []222 current: str | None = None223 lines = body.split("\n")224 i = 0225 while i < len(lines):226 m = re.match(r"^##\s+(.+?)\s*$", lines[i])227 if m:228 current = m.group(1).strip()229 if lines[i].lstrip().startswith("|") and i + 1 < len(lines) and re.match(r"^\s*\|?\s*:?-{2,}", lines[i + 1]):230 out.append(current if current and not current.lower().startswith("what can") else None)231 i += 2232 while i < len(lines) and lines[i].lstrip().startswith("|"):233 i += 1234 continue235 i += 1236 return out237238239_POST = re.compile(r'"slug":\{"_type":"slug","current":"blog/([^"/]+)"\}')240241242def blog_items(content: bytes) -> list[FeedItem]:243 """Post objects live in the RSC payload: …"date":"2026-09-10T…", … "slug":{"current":"blog/<slug>"}, … "title":"…", …"""244 payload = next_flight_payload(content)245 if not payload:246 return []247 items: dict[str, FeedItem] = {}248 matches = list(_POST.finditer(payload))249 for idx, m in enumerate(matches):250 slug = m.group(1)251 start = matches[idx - 1].end() if idx else max(0, m.start() - 20000)252 before = payload[start:m.start()]253 after = payload[m.end():m.end() + 4000]254 date_m = None255 for date_m in re.finditer(r'"date":"(20\d\d-\d\d-\d\d[^"]*)"', before):256 pass257 title_m = re.search(r'"title":"((?:[^"\\]|\\.)*)"', after)258 if not title_m:259 continue260 title = _unescape(title_m.group(1))261 tags = [t for t in re.findall(r'"tag":\["((?:[^"\\]|\\.)*)"\]', after[:3000])]262 url = f"{BLOG}/{slug}"263 if url not in items:264 items[url] = FeedItem(id=url, url=url, title=title[:300], summary=None, published_at=parse_datetime(date_m.group(1)) if date_m else None,265 updated_at=None, categories=[_unescape(t) for t in tags[:5]])266 return sorted(items.values(), key=lambda it: it.published_at.timestamp() if it.published_at else 0, reverse=True)267268269def _unescape(s: str) -> str:270 import json271272 try:273 return json.loads(f'"{s}"').encode("latin-1", errors="ignore").decode("utf-8", errors="ignore") if "Ã" in s or "â" in s else json.loads(f'"{s}"')274 except ValueError:275 return s276277278CONNECTORS: list[Any] = [CohereConnector]279