HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""OpenAI — developer platform docs (served as Markdown: append `.md`) + newsroom RSS.23Sources (tier 1):4 * models catalog → developers.openai.com/api/docs/models.md : every model with its API id + one-line description; per-model pages followed5 * model pages → …/models/<id>.md : Model ID, snapshots, modalities, context window, max output, knowledge cutoff, features6 * pricing → …/pricing.md : Standard / Batch / Flex / Fast tables (short + long context), realtime/audio, image, video, specialized7 * deprecations → …/deprecations.md : dated announcement sections with shutdown tables → deprecation_date / retirement_date / status8 * news RSS → openai.com/news/rss.xml : ANNOUNCEMENT events (article pages answer 403 to crawlers, so they are not followed)910`platform.openai.com/docs/*` permanently redirects to `developers.openai.com/api/docs/*`; the connector targets the final host.11`openai.com/api/pricing/` answers 403 to crawlers — the docs pricing page carries the same tables, so it is not requested at all.12"""13from __future__ import annotations1415import re16from datetime import datetime17from typing import Any1819from aiatlas.ontology.taxonomy import normalize_modalities20from aiatlas.registry import org_ref, provider_ref21from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext22from aiatlas.sdk.extract.dates import parse_datetime23from aiatlas.sdk.extract.markdown import parse_markdown24from aiatlas.sdk.facts import EntityRef, Facts, Target25from aiatlas.sdk.fetch import FetchResult2627from ._common import (28 announcement_events,29 claim_status,30 clean_cell,31 model_ref,32 money,33 month_year,34 normalize_capabilities,35 tokens,36)3738DOCS = "https://developers.openai.com/api/docs"39NEWS_RSS = "https://openai.com/news/rss.xml"40PROVIDER_KEY = "openai"41ID_SCHEME = "openai_model_id"42MAX_MODEL_PAGES = 404344API_ID = re.compile(r"^[a-z0-9][a-z0-9.\-]*$")45CATALOG_ITEM = re.compile(r"^- \[([^\]]+)\]\(/api/docs/models/([^)\s]+?)(?:\.md)?\):\s*(.*)$", re.MULTILINE)46SNAPSHOT_SUFFIX = re.compile(r"-(\d{4}-\d{2}-\d{2}|\d{4})$")47LEGACY_WORDS = re.compile(r"\b(deprecated|older|legacy|previous)\b", re.IGNORECASE)48TIER_LABELS = ("standard", "batch", "flex", "fast mode", "fast", "priority")495051class OpenAIConnector(BaseConnector):52 name = "openai"53 label = "OpenAI — models, pricing, deprecations, news"54 description = "OpenAI developer platform docs (model catalog, model pages, pricing, deprecations) and the OpenAI newsroom feed."55 source_key = "platform.openai.com"56 version = "1"57 parser_version = "1"58 interval_seconds = 360059 min_interval_seconds = 180060 rate_per_min = 1261 tier = 162 priority = 063 expected_min_records = 6064 concurrency = 26566 async def discover(self, ctx: RunContext) -> list[Target]:67 return [68 Target(url=f"{DOCS}/models.md", doc_type="model_docs", key="models", min_bytes=2000, meta={"content_type": "text/markdown"}),69 Target(url=f"{DOCS}/pricing.md", doc_type="pricing", key="pricing", min_bytes=2000, meta={"content_type": "text/markdown"}),70 Target(url=f"{DOCS}/deprecations.md", doc_type="model_docs", key="deprecations", min_bytes=2000, meta={"content_type": "text/markdown"}),71 Target(url=NEWS_RSS, doc_type="feed", key="news", min_bytes=1000),72 ]7374 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:75 facts = Facts()76 org = org_ref("openai")77 facts.entities.append(org)78 key = target.key or ""79 if key == "models" and parsed.markdown:80 self._catalog(facts, org, parsed)81 elif key == "pricing" and parsed.markdown:82 self._pricing(facts, org, parsed)83 elif key == "deprecations" and parsed.markdown:84 self._deprecations(facts, org, parsed, res.fetched_at)85 elif key == "news" and parsed.kind == "feed":86 # openai.com/index/<article> answers 403 to crawlers: events come from the feed only, articles are not followed87 announcement_events(facts, org, parsed.feed_items, source_name="openai.com/news", follow=False)88 facts.document_title = "OpenAI News"89 facts.document_entity = org90 elif target.doc_type == "model_page" and parsed.markdown:91 self._model_page(facts, org, target, parsed)92 return facts9394 # ------------------------------------------------------------------------------------------ catalog95 def _catalog(self, facts: Facts, org: EntityRef, parsed: Parsed) -> None:96 md = parsed.markdown97 assert md98 facts.document_title = "OpenAI models"99 facts.document_entity = org100 featured = md.section(r"^Featured models")101 entries: list[tuple[str, str, str, int]] = [] # (name, id, description, rank)102 seen: set[str] = set()103 for m in CATALOG_ITEM.finditer(md.body):104 name, api_id, desc = m.group(1).strip(), m.group(2).strip(), clean_cell(m.group(3))105 if not API_ID.match(api_id) or api_id in seen:106 continue107 seen.add(api_id)108 rank = 0 if m.group(0) in featured else (2 if LEGACY_WORDS.search(desc) else 1)109 entries.append((name, api_id, desc, rank))110 for name, api_id, desc, _rank in entries:111 ref = _model(facts, org, name, api_id)112 facts.claim(ref, "description", desc)113 facts.claim(ref, "official_url", f"{DOCS}/models/{api_id}")114 low = desc.lower()115 # API-only catalog: weights are only "open" when the docs say so (gpt-oss)116 facts.claim(ref, "openness", "open-weights" if ("open-weight" in low or "open weight" in low) else "proprietary")117 for name, api_id, _desc, _rank in sorted(entries, key=lambda e: e[3])[:MAX_MODEL_PAGES]:118 facts.follow(f"{DOCS}/models/{api_id}.md", doc_type="model_page", key=f"model:{api_id}", meta={"model": name, "api_id": api_id, "content_type": "text/markdown"},119 min_bytes=300, priority=1)120121 # ------------------------------------------------------------------------------------------ model page122 def _model_page(self, facts: Facts, org: EntityRef, target: Target, parsed: Parsed) -> None:123 md = parsed.markdown124 assert md125 title = next((t for lvl, t in md.headings if lvl == 1), None) or target.meta.get("model") or ""126 api_id = target.meta.get("api_id")127 m = re.search(r"Model ID:\s*`([^`]+)`", md.body)128 if m:129 api_id = m.group(1).strip()130 if not title or not api_id:131 return132 ref = _model(facts, org, title, api_id)133 facts.document_title = title134 facts.document_entity = ref135 quotes = [q.strip() for q in re.findall(r"^>\s*(.+)$", md.body, re.MULTILINE) if "llms.txt" not in q]136 if quotes:137 facts.claim(ref, "description", clean_cell(quotes[0]))138 facts.claim(ref, "official_url", f"{DOCS}/models/{api_id}")139 details = md.section(r"^Model details")140 for line in details.split("\n"):141 line = line.strip().lstrip("-").strip()142 low = line.lower()143 if not line:144 continue145 if low.startswith("default snapshot:"):146 snap = clean_cell(line.split(":", 1)[1])147 if snap:148 ref.aliases.append(snap)149 facts.claim(ref, "default_snapshot", snap)150 elif low.startswith("input modalities:"):151 facts.claim(ref, "modalities_input", _modalities(line.split(":", 1)[1]))152 elif low.startswith("output modalities:"):153 facts.claim(ref, "modalities_output", _modalities(line.split(":", 1)[1]))154 elif low.endswith("context window"):155 facts.claim(ref, "context_length", tokens(line), unit="tokens")156 elif low.endswith("max output tokens"):157 facts.claim(ref, "max_output_tokens", tokens(line), unit="tokens")158 elif low.endswith("knowledge cutoff"):159 raw = re.sub(r"knowledge cutoff", "", line, flags=re.IGNORECASE).strip()160 dt = parse_datetime(raw)161 facts.claim(ref, "knowledge_cutoff", month_year(raw) or (f"{dt.year:04d}-{dt.month:02d}" if dt else None))162 mods_in = next((c.value for c in facts.claims if c.entity is ref and c.property == "modalities_input"), None)163 mods_out = next((c.value for c in facts.claims if c.entity is ref and c.property == "modalities_output"), None)164 if mods_in or mods_out:165 facts.claim(ref, "modalities", sorted(set(mods_in or []) | set(mods_out or [])))166 if mods_in and "image" in mods_in:167 facts.claim(ref, "vision", True)168 features = [ln.strip().lstrip("-").strip() for ln in md.section(r"^Supported features").split("\n") if ln.strip().startswith("-")]169 if features:170 facts.claim(ref, "tool_calling", "function_calling" in features)171 facts.claim(ref, "structured_output", "structured_outputs" in features)172 facts.claim(ref, "fine_tuning_available", "fine_tuning" in features)173 if "reasoning" in features:174 facts.claim(ref, "reasoning", True)175 facts.claim(ref, "capabilities", normalize_capabilities(features)) # OpenAI slugs → canonical (structured_outputs → structured_output…)176 facts.claim(ref, "capabilities_raw", features)177 snapshots = [clean_cell(ln.strip().lstrip("-").strip()) for ln in md.section(r"^Snapshots").split("\n") if ln.strip().startswith("-")]178 for s in snapshots:179 if s and API_ID.match(s) and s not in ref.aliases and s != api_id:180 ref.aliases.append(s)181 if snapshots:182 facts.claim(ref, "snapshots", snapshots)183 facts.claim(ref, "openness", "open-weights" if "open-weight" in md.body.lower() else None)184 # provider prices are taken from the pricing page only (complete tuple: batch, cache writes, long context)185186 # ------------------------------------------------------------------------------------------ pricing187 def _pricing(self, facts: Facts, org: EntityRef, parsed: Parsed) -> None:188 md = parsed.markdown189 assert md190 facts.document_title = "OpenAI API pricing"191 provider = provider_ref(PROVIDER_KEY)192 facts.entities.append(provider)193 facts.document_entity = provider194 labels = _table_labels(md.body)195 prices: dict[str, Any] = {} # api id -> PriceObs196197 def obs(api_id: str, display: str | None = None) -> Any:198 """Snapshot rows (gpt-4o-2024-05-13) price the base model (gpt-4o) under their own provider_model_id."""199 if api_id not in prices:200 base = SNAPSHOT_SUFFIX.sub("", api_id)201 ref = _model(facts, org, display or base, base)202 if api_id != base and api_id not in ref.aliases:203 ref.aliases.append(api_id)204 prices[api_id] = facts.price(model=ref, provider=provider, provider_model_id=api_id, meta={"from": "pricing page"})205 return prices[api_id]206207 for i, t in enumerate(md.tables):208 hs = [clean_cell(h).lower() for h in t["headers"]]209 tier = labels[i] if i < len(labels) else "standard"210 if not hs or "training" in hs or hs[0] == "tool":211 continue212 if "short context input" in hs:213 self._flagship_table(t, hs, tier, obs)214 elif hs[:2] == ["model", "modality"]:215 self._modality_table(t, hs, tier, obs)216 elif hs[:2] == ["category", "model"] and tier == "standard":217 for r in t["rows"]:218 api_id, _note = _split_model_cell(r[1])219 if not api_id or len(r) < 5:220 continue221 p = obs(api_id)222 p.input_per_mtok, p.cached_input_per_mtok, p.output_per_mtok = money(r[2]), money(r[3]), money(r[4])223 p.features["category"] = clean_cell(r[0])224 if clean_cell(r[2]).lower() == "free":225 p.features["free"] = True226 elif hs[:2] == ["model", "size"] and "price per second" in hs:227 for r in t["rows"]:228 api_id, _ = _split_model_cell(r[0])229 if api_id and len(r) >= 5:230 p = obs(api_id)231 p.features.setdefault(f"{tier}_per_second_by_size", {})[clean_cell(r[1])] = money(r[4])232 elif hs[:2] == ["model", "use case"]:233 for r in t["rows"]:234 api_id, _ = _split_model_cell(r[0])235 if not api_id or len(r) < 5:236 continue237 p = obs(api_id)238 p.input_per_mtok = p.input_per_mtok if p.input_per_mtok is not None else money(r[2])239 p.output_per_mtok = p.output_per_mtok if p.output_per_mtok is not None else money(r[3])240 p.features["estimated_cost_per_minute"] = money(r[4])241 p.features["use_case"] = clean_cell(r[1])242 elif hs == ["model", "price per minute"]:243 for r in t["rows"]:244 api_id, _ = _split_model_cell(r[0])245 if api_id and len(r) >= 2:246 obs(api_id).features["per_minute"] = money(r[1])247 for p in prices.values():248 p.features = {k: v for k, v in p.features.items() if v not in (None, {}, [])}249 # rows made only of dashes (e.g. a model announced without prices) carry nothing worth storing250 facts.prices = [p for p in facts.prices if any(v is not None for v in p.price_tuple()[:-1]) or p.features]251252 def _flagship_table(self, t: dict[str, Any], hs: list[str], tier: str, obs: Any) -> None:253 col = {h: i for i, h in enumerate(hs)}254 for r in t["rows"]:255 api_id, note = _split_model_cell(r[0])256 if not api_id or len(r) < len(hs):257 continue258 p = obs(api_id)259260 def cell(h: str, row: list[str] = r) -> float | None:261 return money(row[col[h]]) if h in col else None262263 if tier == "standard":264 p.input_per_mtok, p.cached_input_per_mtok, p.output_per_mtok = cell("short context input"), cell("short context cached input"), cell("short context output")265 p.cache_write_per_mtok = cell("short context cache writes")266 for h in ("long context input", "long context cached input", "long context cache writes", "long context output"):267 v = cell(h)268 if v is not None:269 p.features[h.replace(" ", "_") + "_per_mtok"] = v270 if note:271 p.features["context_note"] = note272 elif tier == "batch":273 p.batch_input_per_mtok, p.batch_output_per_mtok = cell("short context input"), cell("short context output")274 else:275 for h in ("short context input", "short context output"):276 v = cell(h)277 if v is not None:278 p.features[f"{tier.replace(' ', '_')}_{h.split()[-1]}_per_mtok"] = v279280 def _modality_table(self, t: dict[str, Any], hs: list[str], tier: str, obs: Any) -> None:281 i_in = next((i for i, h in enumerate(hs) if h == "input"), None)282 i_cached = next((i for i, h in enumerate(hs) if h.startswith("cached")), None)283 i_out = next((i for i, h in enumerate(hs) if h.startswith("output")), None)284 for r in t["rows"]:285 api_id, _ = _split_model_cell(r[0])286 if not api_id or i_in is None or i_out is None or len(r) <= max(i_in, i_out):287 continue288 modality = clean_cell(r[1]).lower()289 p = obs(api_id)290 v_in, v_out = money(r[i_in]), money(r[i_out])291 v_cached = money(r[i_cached]) if i_cached is not None and i_cached < len(r) else None292 if tier == "standard" and modality == "text":293 p.input_per_mtok, p.output_per_mtok, p.cached_input_per_mtok = v_in, v_out, v_cached294 elif tier == "standard":295 p.features[f"{modality}_input_per_mtok"] = v_in296 p.features[f"{modality}_output_per_mtok"] = v_out297 if v_cached is not None:298 p.features[f"{modality}_cached_input_per_mtok"] = v_cached299 if modality == "image" and p.input_per_mtok is None and p.output_per_mtok is None:300 pass301 elif tier == "batch" and modality == "text":302 p.batch_input_per_mtok, p.batch_output_per_mtok = v_in, v_out303 elif tier == "batch":304 p.features[f"batch_{modality}_input_per_mtok"] = v_in305 p.features[f"batch_{modality}_output_per_mtok"] = v_out306307 # ------------------------------------------------------------------------------------------ deprecations308 def _deprecations(self, facts: Facts, org: EntityRef, parsed: Parsed, observed: datetime) -> None:309 """A model can appear in several dated sections (one per retired snapshot): the claims describe its *latest* deprecation round,310 every retired snapshot is listed once, and the API id is left to the catalog (this page names families and snapshots)."""311 md = parsed.markdown312 assert md313 facts.document_title = "OpenAI deprecations"314 facts.document_entity = org315 rounds: dict[str, dict[str, Any]] = {}316 for level, heading in md.headings:317 m = re.match(r"^(\d{4}-\d{2}-\d{2}):\s*(.+)$", heading)318 if level != 3 or not m:319 continue320 announced = m.group(1)321 section = parse_markdown(md.section(re.escape(heading)))322 for t in section.tables:323 hs = [clean_cell(h).lower() for h in t["headers"]]324 i_date = next((i for i, h in enumerate(hs) if "shutdown" in h), None)325 i_model = next((i for i, h in enumerate(hs) if "model" in h), None)326 i_repl = next((i for i, h in enumerate(hs) if "replacement" in h or "substitute" in h), None)327 if i_date is None or i_model is None:328 continue329 for r in t["rows"]:330 if len(r) <= max(i_date, i_model):331 continue332 ids = _ids_in_cell(r[i_model])333 if not ids:334 continue335 base = SNAPSHOT_SUFFIX.sub("", ids[0])336 shutdown, tentative = _shutdown_date(r[i_date])337 repl = _ids_in_cell(r[i_repl]) if i_repl is not None and i_repl < len(r) else []338 entry = rounds.setdefault(base, {"snapshots": set(), "aliases": set(), "rounds": [], "dated": False})339 entry["dated"] = entry["dated"] or ids[0] != base340 entry["snapshots"].update(i for i in ids[:1] if i != base)341 entry["aliases"].update(i for i in ids if i != base)342 entry["rounds"].append({"announced": announced, "shutdown": shutdown, "tentative": tentative,343 "replacement": SNAPSHOT_SUFFIX.sub("", repl[0]) if repl else None, "replacement_raw": repl[0] if repl else None})344 # `gpt-4o-audio` (a bare family label, never a dated snapshot) and `gpt-4o-audio-preview-<date>` name one lineage whose catalog id345 # is the -preview one; `o1` / `o1-preview` or `gpt-4-turbo` / `gpt-4-turbo-preview` are distinct models (each has its own snapshots)346 for plain in [k for k in rounds if f"{k}-preview" in rounds and not rounds[k]["dated"]]:347 target_entry = rounds[f"{plain}-preview"]348 target_entry["rounds"] += rounds[plain]["rounds"]349 target_entry["snapshots"] |= rounds[plain]["snapshots"]350 target_entry["aliases"] |= rounds[plain]["aliases"] | {plain}351 del rounds[plain]352 today = observed.date().isoformat()353 for base, entry in rounds.items():354 ref = _model(facts, org, base, base, claim_id=False)355 for extra in sorted(entry["aliases"]):356 if extra not in ref.aliases:357 ref.aliases.append(extra)358 latest = max(entry["rounds"], key=lambda x: (x["shutdown"] or "", x["announced"]))359 facts.claim(ref, "deprecation_date", latest["announced"])360 if latest["shutdown"]:361 facts.claim(ref, "retirement_date", latest["shutdown"])362 facts.claim(ref, "retirement_tentative", latest["tentative"])363 claim_status(facts, ref, "retired" if latest["shutdown"] < today else "deprecated")364 else:365 claim_status(facts, ref, "deprecated")366 if entry["snapshots"]:367 facts.claim(ref, "retired_snapshots", sorted(entry["snapshots"]))368 if latest["replacement"] and latest["replacement"] != base:369 new = _model(facts, org, latest["replacement"], latest["replacement"], claim_id=False)370 if latest["replacement_raw"] != latest["replacement"] and latest["replacement_raw"] not in new.aliases:371 new.aliases.append(latest["replacement_raw"])372 facts.relate(ref, "superseded_by", new, attributes={"announced": latest["announced"], "shutdown": latest["shutdown"]})373374375# ---------------------------------------------------------------------------------------------- helpers376def _model(facts: Facts, org: EntityRef, name: str, api_id: str, *, claim_id: bool = True) -> EntityRef:377 """One EntityRef per API id inside a Facts (so claims from several tables land on the same ref)."""378 for e in facts.entities:379 if e.entity_type == "model" and e.identifiers.get(ID_SCHEME) == api_id:380 return e381 aliases = [api_id] if api_id != name else []382 ref = model_ref(facts, name, org, api_id=api_id, provider_key=PROVIDER_KEY, aliases=aliases)383 if claim_id:384 facts.claim(ref, "api_model_id", api_id)385 return ref386387388def _modalities(cell: str) -> list[str]:389 return normalize_modalities(cell)390391392def _split_model_cell(cell: str) -> tuple[str | None, str | None]:393 """'gpt-5.5 (<272K context length)' → ('gpt-5.5', '<272K context length'); 'Whisper' → (None, None)."""394 c = clean_cell(cell)395 m = re.match(r"^([^\s(]+)\s*(?:\((.*)\))?$", c)396 if not m:397 return None, None398 api_id = m.group(1).strip()399 if not API_ID.match(api_id):400 return None, None401 return api_id, (m.group(2).strip() if m.group(2) else None)402403404def _ids_in_cell(cell: str) -> list[str]:405 ids: list[str] = []406 for tok in re.split(r"[|,]| or ", clean_cell(cell)):407 tok = tok.strip().strip("`*").split(" ")[0].strip("`*")408 if tok and API_ID.match(tok) and not tok.startswith("ft-") and tok not in ids and tok != "---":409 ids.append(tok)410 return ids411412413def _shutdown_date(cell: str) -> tuple[str | None, bool]:414 c = clean_cell(cell).replace("‑", "-").replace("‐", "-")415 tentative = "earliest" in c.lower() or "not sooner" in c.lower()416 m = re.search(r"(\d{4}-\d{2}-\d{2}|[A-Z][a-z]+ \d{1,2}, \d{4})", c)417 dt = parse_datetime(m.group(1)) if m else None418 return (dt.date().isoformat() if dt else None), tentative419420421def _table_labels(body: str) -> list[str]:422 """Pricing tier ('standard', 'batch', 'flex', 'fast mode') preceding each Markdown table, in table order."""423 labels: list[str] = []424 lines = body.split("\n")425 current = "standard"426 i = 0427 while i < len(lines):428 line = lines[i].strip()429 low = line.lower()430 if low in TIER_LABELS:431 current = "fast mode" if low == "fast" else low432 elif re.match(r"^[A-Z][a-z]+ models?$", line) or re.match(r"^(Tools|Finetuning|Transcription models)$", line):433 current = "standard"434 if line.startswith("|") and i + 1 < len(lines) and re.match(r"^\s*\|?\s*:?-{2,}", lines[i + 1]):435 labels.append(current)436 i += 2437 while i < len(lines) and lines[i].strip().startswith("|"):438 i += 1439 continue440 i += 1441 return labels442443444CONNECTORS = [OpenAIConnector]445