1"""OpenAI — developer platform docs (served as Markdown: append `.md`) + newsroom RSS.23Sources (tier 1):4 * models catalog → developers.openai.com/api/docs/models.md : every model with its API id + one-line description; per-model pages followed5 * model pages → …/models/<id>.md : Model ID, snapshots, modalities, context window, max output, knowledge cutoff, features6 * pricing → …/pricing.md : Standard / Batch / Flex / Fast tables (short + long context), realtime/audio, image, video, specialized7 * deprecations → …/deprecations.md : dated announcement sections with shutdown tables → deprecation_date / retirement_date / status8 * news RSS → openai.com/news/rss.xml : ANNOUNCEMENT events (article pages answer 403 to crawlers, so they are not followed)910`platform.openai.com/docs/*` permanently redirects to `developers.openai.com/api/docs/*`; the connector targets the final host.11`openai.com/api/pricing/` answers 403 to crawlers — the docs pricing page carries the same tables, so it is not requested at all.12"""13from __future__ import annotations1415import re16from datetime import datetime17from typing import Any1819from aiatlas.registry import org_ref, provider_ref20from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext21from aiatlas.sdk.extract.dates import parse_datetime22from aiatlas.sdk.extract.markdown import parse_markdown23from aiatlas.sdk.facts import EntityRef, Facts, Target24from aiatlas.sdk.fetch import FetchResult2526from ._common import announcement_events, clean_cell, model_ref, money, month_year, tokens2728DOCS = "https://developers.openai.com/api/docs"29NEWS_RSS = "https://openai.com/news/rss.xml"30PROVIDER_KEY = "openai"31ID_SCHEME = "openai_model_id"32MAX_MODEL_PAGES = 403334API_ID = re.compile(r"^[a-z0-9][a-z0-9.\-]*$")35CATALOG_ITEM = re.compile(r"^- \[([^\]]+)\]\(/api/docs/models/([^)\s]+?)(?:\.md)?\):\s*(.*)$", re.MULTILINE)36SNAPSHOT_SUFFIX = re.compile(r"-(\d{4}-\d{2}-\d{2}|\d{4})$")37LEGACY_WORDS = re.compile(r"\b(deprecated|older|legacy|previous)\b", re.IGNORECASE)38TIER_LABELS = ("standard", "batch", "flex", "fast mode", "fast", "priority")394041class OpenAIConnector(BaseConnector):42 name = "openai"43 label = "OpenAI — models, pricing, deprecations, news"44 description = "OpenAI developer platform docs (model catalog, model pages, pricing, deprecations) and the OpenAI newsroom feed."45 source_key = "platform.openai.com"46 version = "1"47 parser_version = "1"48 interval_seconds = 360049 min_interval_seconds = 180050 rate_per_min = 1251 tier = 152 priority = 053 expected_min_records = 6054 concurrency = 25556 async def discover(self, ctx: RunContext) -> list[Target]:57 return [58 Target(url=f"{DOCS}/models.md", doc_type="model_docs", key="models", min_bytes=2000, meta={"content_type": "text/markdown"}),59 Target(url=f"{DOCS}/pricing.md", doc_type="pricing", key="pricing", min_bytes=2000, meta={"content_type": "text/markdown"}),60 Target(url=f"{DOCS}/deprecations.md", doc_type="model_docs", key="deprecations", min_bytes=2000, meta={"content_type": "text/markdown"}),61 Target(url=NEWS_RSS, doc_type="feed", key="news", min_bytes=1000),62 ]6364 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:65 facts = Facts()66 org = org_ref("openai")67 facts.entities.append(org)68 key = target.key or ""69 if key == "models" and parsed.markdown:70 self._catalog(facts, org, parsed)71 elif key == "pricing" and parsed.markdown:72 self._pricing(facts, org, parsed)73 elif key == "deprecations" and parsed.markdown:74 self._deprecations(facts, org, parsed, res.fetched_at)75 elif key == "news" and parsed.kind == "feed":76 # openai.com/index/<article> answers 403 to crawlers: events come from the feed only, articles are not followed77 announcement_events(facts, org, parsed.feed_items, source_name="openai.com/news", follow=False)78 facts.document_title = "OpenAI News"79 facts.document_entity = org80 elif target.doc_type == "model_page" and parsed.markdown:81 self._model_page(facts, org, target, parsed)82 return facts8384 # ------------------------------------------------------------------------------------------ catalog85 def _catalog(self, facts: Facts, org: EntityRef, parsed: Parsed) -> None:86 md = parsed.markdown87 assert md88 facts.document_title = "OpenAI models"89 facts.document_entity = org90 featured = md.section(r"^Featured models")91 entries: list[tuple[str, str, str, int]] = [] # (name, id, description, rank)92 seen: set[str] = set()93 for m in CATALOG_ITEM.finditer(md.body):94 name, api_id, desc = m.group(1).strip(), m.group(2).strip(), clean_cell(m.group(3))95 if not API_ID.match(api_id) or api_id in seen:96 continue97 seen.add(api_id)98 rank = 0 if m.group(0) in featured else (2 if LEGACY_WORDS.search(desc) else 1)99 entries.append((name, api_id, desc, rank))100 for name, api_id, desc, _rank in entries:101 ref = _model(facts, org, name, api_id)102 facts.claim(ref, "description", desc)103 facts.claim(ref, "official_url", f"{DOCS}/models/{api_id}")104 low = desc.lower()105 # API-only catalog: weights are only "open" when the docs say so (gpt-oss)106 facts.claim(ref, "openness", "open-weights" if ("open-weight" in low or "open weight" in low) else "proprietary")107 for name, api_id, _desc, _rank in sorted(entries, key=lambda e: e[3])[:MAX_MODEL_PAGES]:108 facts.follow(f"{DOCS}/models/{api_id}.md", doc_type="model_page", key=f"model:{api_id}", meta={"model": name, "api_id": api_id, "content_type": "text/markdown"},109 min_bytes=300, priority=1)110111 # ------------------------------------------------------------------------------------------ model page112 def _model_page(self, facts: Facts, org: EntityRef, target: Target, parsed: Parsed) -> None:113 md = parsed.markdown114 assert md115 title = next((t for lvl, t in md.headings if lvl == 1), None) or target.meta.get("model") or ""116 api_id = target.meta.get("api_id")117 m = re.search(r"Model ID:\s*`([^`]+)`", md.body)118 if m:119 api_id = m.group(1).strip()120 if not title or not api_id:121 return122 ref = _model(facts, org, title, api_id)123 facts.document_title = title124 facts.document_entity = ref125 quotes = [q.strip() for q in re.findall(r"^>\s*(.+)$", md.body, re.MULTILINE) if "llms.txt" not in q]126 if quotes:127 facts.claim(ref, "description", clean_cell(quotes[0]))128 facts.claim(ref, "official_url", f"{DOCS}/models/{api_id}")129 details = md.section(r"^Model details")130 for line in details.split("\n"):131 line = line.strip().lstrip("-").strip()132 low = line.lower()133 if not line:134 continue135 if low.startswith("default snapshot:"):136 snap = clean_cell(line.split(":", 1)[1])137 if snap:138 ref.aliases.append(snap)139 facts.claim(ref, "default_snapshot", snap)140 elif low.startswith("input modalities:"):141 facts.claim(ref, "modalities_input", _modalities(line.split(":", 1)[1]))142 elif low.startswith("output modalities:"):143 facts.claim(ref, "modalities_output", _modalities(line.split(":", 1)[1]))144 elif low.endswith("context window"):145 facts.claim(ref, "context_length", tokens(line), unit="tokens")146 elif low.endswith("max output tokens"):147 facts.claim(ref, "max_output_tokens", tokens(line), unit="tokens")148 elif low.endswith("knowledge cutoff"):149 raw = re.sub(r"knowledge cutoff", "", line, flags=re.IGNORECASE).strip()150 dt = parse_datetime(raw)151 facts.claim(ref, "knowledge_cutoff", month_year(raw) or (f"{dt.year:04d}-{dt.month:02d}" if dt else None))152 mods_in = next((c.value for c in facts.claims if c.entity is ref and c.property == "modalities_input"), None)153 mods_out = next((c.value for c in facts.claims if c.entity is ref and c.property == "modalities_output"), None)154 if mods_in or mods_out:155 facts.claim(ref, "modalities", sorted(set(mods_in or []) | set(mods_out or [])))156 if mods_in and "image" in mods_in:157 facts.claim(ref, "vision", True)158 features = [ln.strip().lstrip("-").strip() for ln in md.section(r"^Supported features").split("\n") if ln.strip().startswith("-")]159 if features:160 facts.claim(ref, "tool_calling", "function_calling" in features)161 facts.claim(ref, "structured_output", "structured_outputs" in features)162 facts.claim(ref, "fine_tuning_available", "fine_tuning" in features)163 if "reasoning" in features:164 facts.claim(ref, "reasoning", True)165 facts.claim(ref, "capabilities", features)166 snapshots = [clean_cell(ln.strip().lstrip("-").strip()) for ln in md.section(r"^Snapshots").split("\n") if ln.strip().startswith("-")]167 for s in snapshots:168 if s and API_ID.match(s) and s not in ref.aliases and s != api_id:169 ref.aliases.append(s)170 if snapshots:171 facts.claim(ref, "snapshots", snapshots)172 facts.claim(ref, "openness", "open-weights" if "open-weight" in md.body.lower() else None)173 # provider prices are taken from the pricing page only (complete tuple: batch, cache writes, long context)174175 # ------------------------------------------------------------------------------------------ pricing176 def _pricing(self, facts: Facts, org: EntityRef, parsed: Parsed) -> None:177 md = parsed.markdown178 assert md179 facts.document_title = "OpenAI API pricing"180 provider = provider_ref(PROVIDER_KEY)181 facts.entities.append(provider)182 facts.document_entity = provider183 labels = _table_labels(md.body)184 prices: dict[str, Any] = {} # api id -> PriceObs185186 def obs(api_id: str, display: str | None = None) -> Any:187 """Snapshot rows (gpt-4o-2024-05-13) price the base model (gpt-4o) under their own provider_model_id."""188 if api_id not in prices:189 base = SNAPSHOT_SUFFIX.sub("", api_id)190 ref = _model(facts, org, display or base, base)191 if api_id != base and api_id not in ref.aliases:192 ref.aliases.append(api_id)193 prices[api_id] = facts.price(model=ref, provider=provider, provider_model_id=api_id, meta={"from": "pricing page"})194 return prices[api_id]195196 for i, t in enumerate(md.tables):197 hs = [clean_cell(h).lower() for h in t["headers"]]198 tier = labels[i] if i < len(labels) else "standard"199 if not hs or "training" in hs or hs[0] == "tool":200 continue201 if "short context input" in hs:202 self._flagship_table(t, hs, tier, obs)203 elif hs[:2] == ["model", "modality"]:204 self._modality_table(t, hs, tier, obs)205 elif hs[:2] == ["category", "model"] and tier == "standard":206 for r in t["rows"]:207 api_id, _note = _split_model_cell(r[1])208 if not api_id or len(r) < 5:209 continue210 p = obs(api_id)211 p.input_per_mtok, p.cached_input_per_mtok, p.output_per_mtok = money(r[2]), money(r[3]), money(r[4])212 p.features["category"] = clean_cell(r[0])213 if clean_cell(r[2]).lower() == "free":214 p.features["free"] = True215 elif hs[:2] == ["model", "size"] and "price per second" in hs:216 for r in t["rows"]:217 api_id, _ = _split_model_cell(r[0])218 if api_id and len(r) >= 5:219 p = obs(api_id)220 p.features.setdefault(f"{tier}_per_second_by_size", {})[clean_cell(r[1])] = money(r[4])221 elif hs[:2] == ["model", "use case"]:222 for r in t["rows"]:223 api_id, _ = _split_model_cell(r[0])224 if not api_id or len(r) < 5:225 continue226 p = obs(api_id)227 p.input_per_mtok = p.input_per_mtok if p.input_per_mtok is not None else money(r[2])228 p.output_per_mtok = p.output_per_mtok if p.output_per_mtok is not None else money(r[3])229 p.features["estimated_cost_per_minute"] = money(r[4])230 p.features["use_case"] = clean_cell(r[1])231 elif hs == ["model", "price per minute"]:232 for r in t["rows"]:233 api_id, _ = _split_model_cell(r[0])234 if api_id and len(r) >= 2:235 obs(api_id).features["per_minute"] = money(r[1])236 for p in prices.values():237 p.features = {k: v for k, v in p.features.items() if v not in (None, {}, [])}238 # rows made only of dashes (e.g. a model announced without prices) carry nothing worth storing239 facts.prices = [p for p in facts.prices if any(v is not None for v in p.price_tuple()[:-1]) or p.features]240241 def _flagship_table(self, t: dict[str, Any], hs: list[str], tier: str, obs: Any) -> None:242 col = {h: i for i, h in enumerate(hs)}243 for r in t["rows"]:244 api_id, note = _split_model_cell(r[0])245 if not api_id or len(r) < len(hs):246 continue247 p = obs(api_id)248249 def cell(h: str, row: list[str] = r) -> float | None:250 return money(row[col[h]]) if h in col else None251252 if tier == "standard":253 p.input_per_mtok, p.cached_input_per_mtok, p.output_per_mtok = cell("short context input"), cell("short context cached input"), cell("short context output")254 p.cache_write_per_mtok = cell("short context cache writes")255 for h in ("long context input", "long context cached input", "long context cache writes", "long context output"):256 v = cell(h)257 if v is not None:258 p.features[h.replace(" ", "_") + "_per_mtok"] = v259 if note:260 p.features["context_note"] = note261 elif tier == "batch":262 p.batch_input_per_mtok, p.batch_output_per_mtok = cell("short context input"), cell("short context output")263 else:264 for h in ("short context input", "short context output"):265 v = cell(h)266 if v is not None:267 p.features[f"{tier.replace(' ', '_')}_{h.split()[-1]}_per_mtok"] = v268269 def _modality_table(self, t: dict[str, Any], hs: list[str], tier: str, obs: Any) -> None:270 i_in = next((i for i, h in enumerate(hs) if h == "input"), None)271 i_cached = next((i for i, h in enumerate(hs) if h.startswith("cached")), None)272 i_out = next((i for i, h in enumerate(hs) if h.startswith("output")), None)273 for r in t["rows"]:274 api_id, _ = _split_model_cell(r[0])275 if not api_id or i_in is None or i_out is None or len(r) <= max(i_in, i_out):276 continue277 modality = clean_cell(r[1]).lower()278 p = obs(api_id)279 v_in, v_out = money(r[i_in]), money(r[i_out])280 v_cached = money(r[i_cached]) if i_cached is not None and i_cached < len(r) else None281 if tier == "standard" and modality == "text":282 p.input_per_mtok, p.output_per_mtok, p.cached_input_per_mtok = v_in, v_out, v_cached283 elif tier == "standard":284 p.features[f"{modality}_input_per_mtok"] = v_in285 p.features[f"{modality}_output_per_mtok"] = v_out286 if v_cached is not None:287 p.features[f"{modality}_cached_input_per_mtok"] = v_cached288 if modality == "image" and p.input_per_mtok is None and p.output_per_mtok is None:289 pass290 elif tier == "batch" and modality == "text":291 p.batch_input_per_mtok, p.batch_output_per_mtok = v_in, v_out292 elif tier == "batch":293 p.features[f"batch_{modality}_input_per_mtok"] = v_in294 p.features[f"batch_{modality}_output_per_mtok"] = v_out295296 # ------------------------------------------------------------------------------------------ deprecations297 def _deprecations(self, facts: Facts, org: EntityRef, parsed: Parsed, observed: datetime) -> None:298 """A model can appear in several dated sections (one per retired snapshot): the claims describe its *latest* deprecation round,299 every retired snapshot is listed once, and the API id is left to the catalog (this page names families and snapshots)."""300 md = parsed.markdown301 assert md302 facts.document_title = "OpenAI deprecations"303 facts.document_entity = org304 rounds: dict[str, dict[str, Any]] = {}305 for level, heading in md.headings:306 m = re.match(r"^(\d{4}-\d{2}-\d{2}):\s*(.+)$", heading)307 if level != 3 or not m:308 continue309 announced = m.group(1)310 section = parse_markdown(md.section(re.escape(heading)))311 for t in section.tables:312 hs = [clean_cell(h).lower() for h in t["headers"]]313 i_date = next((i for i, h in enumerate(hs) if "shutdown" in h), None)314 i_model = next((i for i, h in enumerate(hs) if "model" in h), None)315 i_repl = next((i for i, h in enumerate(hs) if "replacement" in h or "substitute" in h), None)316 if i_date is None or i_model is None:317 continue318 for r in t["rows"]:319 if len(r) <= max(i_date, i_model):320 continue321 ids = _ids_in_cell(r[i_model])322 if not ids:323 continue324 base = SNAPSHOT_SUFFIX.sub("", ids[0])325 shutdown, tentative = _shutdown_date(r[i_date])326 repl = _ids_in_cell(r[i_repl]) if i_repl is not None and i_repl < len(r) else []327 entry = rounds.setdefault(base, {"snapshots": set(), "aliases": set(), "rounds": [], "dated": False})328 entry["dated"] = entry["dated"] or ids[0] != base329 entry["snapshots"].update(i for i in ids[:1] if i != base)330 entry["aliases"].update(i for i in ids if i != base)331 entry["rounds"].append({"announced": announced, "shutdown": shutdown, "tentative": tentative,332 "replacement": SNAPSHOT_SUFFIX.sub("", repl[0]) if repl else None, "replacement_raw": repl[0] if repl else None})333 # `gpt-4o-audio` (a bare family label, never a dated snapshot) and `gpt-4o-audio-preview-<date>` name one lineage whose catalog id334 # is the -preview one; `o1` / `o1-preview` or `gpt-4-turbo` / `gpt-4-turbo-preview` are distinct models (each has its own snapshots)335 for plain in [k for k in rounds if f"{k}-preview" in rounds and not rounds[k]["dated"]]:336 target_entry = rounds[f"{plain}-preview"]337 target_entry["rounds"] += rounds[plain]["rounds"]338 target_entry["snapshots"] |= rounds[plain]["snapshots"]339 target_entry["aliases"] |= rounds[plain]["aliases"] | {plain}340 del rounds[plain]341 today = observed.date().isoformat()342 for base, entry in rounds.items():343 ref = _model(facts, org, base, base, claim_id=False)344 for extra in sorted(entry["aliases"]):345 if extra not in ref.aliases:346 ref.aliases.append(extra)347 latest = max(entry["rounds"], key=lambda x: (x["shutdown"] or "", x["announced"]))348 facts.claim(ref, "deprecation_date", latest["announced"])349 if latest["shutdown"]:350 facts.claim(ref, "retirement_date", latest["shutdown"])351 facts.claim(ref, "retirement_tentative", latest["tentative"])352 facts.claim(ref, "status", "retired" if latest["shutdown"] < today else "deprecated")353 else:354 facts.claim(ref, "status", "deprecated")355 if entry["snapshots"]:356 facts.claim(ref, "retired_snapshots", sorted(entry["snapshots"]))357 if latest["replacement"] and latest["replacement"] != base:358 new = _model(facts, org, latest["replacement"], latest["replacement"], claim_id=False)359 if latest["replacement_raw"] != latest["replacement"] and latest["replacement_raw"] not in new.aliases:360 new.aliases.append(latest["replacement_raw"])361 facts.relate(ref, "superseded_by", new, attributes={"announced": latest["announced"], "shutdown": latest["shutdown"]})362363364# ---------------------------------------------------------------------------------------------- helpers365def _model(facts: Facts, org: EntityRef, name: str, api_id: str, *, claim_id: bool = True) -> EntityRef:366 """One EntityRef per API id inside a Facts (so claims from several tables land on the same ref)."""367 for e in facts.entities:368 if e.entity_type == "model" and e.identifiers.get(ID_SCHEME) == api_id:369 return e370 aliases = [api_id] if api_id != name else []371 ref = model_ref(facts, name, org, api_id=api_id, provider_key=PROVIDER_KEY, aliases=aliases)372 if claim_id:373 facts.claim(ref, "api_model_id", api_id)374 return ref375376377def _modalities(cell: str) -> list[str]:378 out = []379 for part in re.split(r"[,/]| and ", cell.lower()):380 part = part.strip()381 if part in ("text", "image", "audio", "video", "code", "embedding"):382 out.append(part)383 return out384385386def _split_model_cell(cell: str) -> tuple[str | None, str | None]:387 """'gpt-5.5 (<272K context length)' → ('gpt-5.5', '<272K context length'); 'Whisper' → (None, None)."""388 c = clean_cell(cell)389 m = re.match(r"^([^\s(]+)\s*(?:\((.*)\))?$", c)390 if not m:391 return None, None392 api_id = m.group(1).strip()393 if not API_ID.match(api_id):394 return None, None395 return api_id, (m.group(2).strip() if m.group(2) else None)396397398def _ids_in_cell(cell: str) -> list[str]:399 ids: list[str] = []400 for tok in re.split(r"[|,]| or ", clean_cell(cell)):401 tok = tok.strip().strip("`*").split(" ")[0].strip("`*")402 if tok and API_ID.match(tok) and not tok.startswith("ft-") and tok not in ids and tok != "---":403 ids.append(tok)404 return ids405406407def _shutdown_date(cell: str) -> tuple[str | None, bool]:408 c = clean_cell(cell).replace("‑", "-").replace("‐", "-")409 tentative = "earliest" in c.lower() or "not sooner" in c.lower()410 m = re.search(r"(\d{4}-\d{2}-\d{2}|[A-Z][a-z]+ \d{1,2}, \d{4})", c)411 dt = parse_datetime(m.group(1)) if m else None412 return (dt.date().isoformat() if dt else None), tentative413414415def _table_labels(body: str) -> list[str]:416 """Pricing tier ('standard', 'batch', 'flex', 'fast mode') preceding each Markdown table, in table order."""417 labels: list[str] = []418 lines = body.split("\n")419 current = "standard"420 i = 0421 while i < len(lines):422 line = lines[i].strip()423 low = line.lower()424 if low in TIER_LABELS:425 current = "fast mode" if low == "fast" else low426 elif re.match(r"^[A-Z][a-z]+ models?$", line) or re.match(r"^(Tools|Finetuning|Transcription models)$", line):427 current = "standard"428 if line.startswith("|") and i + 1 < len(lines) and re.match(r"^\s*\|?\s*:?-{2,}", lines[i + 1]):429 labels.append(current)430 i += 2431 while i < len(lines) and lines[i].strip().startswith("|"):432 i += 1433 continue434 i += 1435 return labels436437438CONNECTORS = [OpenAIConnector]439