HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Official provider pricing pages that render server-side (tier 1). One connector class per provider domain so that every document,2claim and price row carries the right `source` (the SDK binds one `source_key` per connector) — module name kept as `provider_pricing`.34 * groq_pricing https://console.groq.com/docs/models — HTML tables (Production / Systems / Preview): model id (`div[id]`), name,5 speed, "$0.15 input $0.60 output" per 1M tokens, rate limits, context window, max completion tokens.6 https://groq.com/pricing is a client-rendered Next.js page with no price in the HTML → fetched with `escalate=True`7 (only used when a browser/Scrapfly key is configured) and otherwise recorded as "no data" in the run log.8 * together_pricing https://www.together.ai/pricing — Webflow tables: serverless (input / cached / output per 1M tokens) and batch9 tables share the model page slug (`/models/<slug>`), merged into one price row per model.10 * fireworks_pricing https://docs.fireworks.ai/serverless/pricing.md — the docs platform serves Markdown: "Standard" and "Priority"11 cells are `input / cached input / output`; Fast/US variants are separate rows sharing the base model link.12 https://fireworks.ai/pricing only lists size-based embedding tiers and GPU hours, so it is not fetched.13"""14from __future__ import annotations1516import re17from typing import Any1819from aiatlas.connectors._identity import family_ref, model_identity, org_ref_in20from aiatlas.registry import org_by_hf, organizations, provider_ref21from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext22from aiatlas.sdk.extract.numbers import parse_context_length23from aiatlas.sdk.facts import EntityRef, Facts, Target24from aiatlas.sdk.fetch import FetchResult2526MONEY = re.compile(r"\$\s*(\d+(?:\.\d+)?)")27INPUT_OUTPUT = re.compile(r"\$\s*(\d+(?:\.\d+)?)\s*input\s*\$\s*(\d+(?:\.\d+)?)\s*output", re.IGNORECASE)282930def _money(s: str) -> float | None:31 m = MONEY.search((s or "").replace(",", ""))32 return float(m.group(1)) if m else None333435def _org_for(facts: Facts, *candidates: str | None) -> EntityRef | None:36 """Registry organization from a vendor slug / display word: exact hf_org, key, name or alias match only — never a guess."""37 for c in candidates:38 if not c:39 continue40 low = c.strip().lower()41 known = org_by_hf(low)42 if known:43 return org_ref_in(facts, known["key"])44 for key, o in organizations().items():45 names = {key, o["name"].lower(), *(a.lower() for a in o.get("aliases", []))}46 if low in names:47 return org_ref_in(facts, "meta-ai" if key == "meta" and "meta-ai" in organizations() else key) # models come from the lab, not the holding48 return None495051def _org_for_id(facts: Facts, model_id: str | None, *fallback_words: str | None) -> EntityRef | None:52 """Developer organisation of a provider's model id (`openai/gpt-oss-120b`, `fireworks/kimi-k3`, `minimax-m3`): first-party prefix or53 family word through the shared identity helper, then an exact registry match on the display words."""54 ident = model_identity(model_id) if model_id else None55 if ident and ident.org_key:56 return org_ref_in(facts, ident.org_key)57 return _org_for(facts, *fallback_words)585960class _PricingBase(BaseConnector):61 version = "2"62 parser_version = "2"63 interval_seconds = 6 * 360064 min_interval_seconds = 3 * 360065 max_interval_seconds = 3 * 8640066 tier = 167 priority = 168 concurrency = 169 provider_key = ""7071 def _model(self, facts: Facts, name: str, *, ids: dict[str, str], org: EntityRef | None, aliases: list[str]) -> EntityRef:72 for e in facts.entities:73 if e.entity_type == "model" and ids and any(e.identifiers.get(k) == v for k, v in ids.items()):74 return e75 ref = facts.entity("model", name, identifiers=ids, organization=org, aliases=[a for a in dict.fromkeys(aliases) if a and a != name],76 family=family_ref(name, org), identity_confidence="medium")77 if org:78 facts.relate(org, "develops", ref)79 return ref808182# ================================================================================================ Groq83class GroqPricingConnector(_PricingBase):84 name = "groq_pricing"85 label = "GroqCloud — supported models, limits and prices"86 description = "console.groq.com/docs/models tables (server-rendered) and the marketing pricing page (client-rendered, escalation only)."87 source_key = "groq.com"88 rate_per_min = 1289 expected_min_records = 890 provider_key = "groq"9192 async def discover(self, ctx: RunContext) -> list[Target]:93 return [Target(url="https://console.groq.com/docs/models", doc_type="model_docs", key="groq_models", min_bytes=20000, priority=1),94 Target(url="https://groq.com/pricing", doc_type="pricing", key="groq_pricing", min_bytes=5000, priority=3, escalate=True)]9596 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:97 facts = Facts()98 provider = provider_ref(self.provider_key)99 facts.entities.append(provider)100 facts.document_entity = provider101 if not parsed.html:102 return facts103 if target.key == "groq_pricing":104 if "$" not in parsed.html.text:105 ctx.log.warning("groq.com/pricing has no server-rendered prices (client-side app); nothing extracted", extra={"url": target.url})106 facts.document_title = parsed.html.title107 return facts108 html = parsed.html109 section = "production"110 for node in html.tree.body.traverse() if html.tree and html.tree.body else []:111 if node.tag == "h2":112 t = node.text(strip=True).lower()113 section = "preview" if "preview" in t else "deprecated" if "deprecated" in t else "system" if "system" in t else "production" if "production" in t else section114 elif node.tag == "tr" and node.css_first("td"):115 self._row(facts, provider, node, section)116 facts.document_title = html.title117 return facts118119 def _row(self, facts: Facts, provider: EntityRef, tr: Any, section: str) -> None:120 cells = tr.css("td")121 if len(cells) < 6:122 return123 first = cells[0]124 id_node = first.css_first("div[id]")125 model_id = (id_node.attributes.get("id") if id_node else None) or (first.css_first("span.font-mono").text(strip=True) if first.css_first("span.font-mono") else None)126 link = first.css_first("a")127 name = link.text(strip=True) if link else None128 if not model_id or not name:129 return130 if section == "deprecated":131 return132 img = first.css_first("img[alt]")133 vendor = model_id.split("/")[0] if "/" in model_id else None134 org = _org_for_id(facts, model_id, vendor, img.attributes.get("alt") if img else None)135 ref = self._model(facts, name, ids={"groq_model_id": model_id}, org=org, aliases=[model_id, model_id.split("/")[-1]])136 facts.claim(ref, "groq_model_id", model_id)137 texts = [c.text(separator=" ", strip=True) for c in cells]138 ctx_len = parse_context_length(texts[4]) if texts[4] not in ("-", "") else None139 max_out = parse_context_length(texts[5]) if texts[5] not in ("-", "") else None140 badge = first.css_first("span.select-none")141 tier_label = badge.text(strip=True).lower() if badge else None142 features: dict[str, Any] = {"groq_section": section}143 if tier_label:144 features["groq_tier"] = tier_label145 speed = texts[1].replace(",", "")146 if speed.isdigit():147 features["output_tokens_per_second"] = int(speed)148 if texts[3] and texts[3] != "-":149 features["rate_limit_developer_plan"] = texts[3]150 m = INPUT_OUTPUT.search(texts[2])151 if ctx_len:152 features["context_length"] = ctx_len # Groq's deployment limit — the model's own context is claimed by the lab's docs153 if max_out:154 features["max_completion_tokens"] = max_out155 if m:156 facts.price(model=ref, provider=provider, provider_model_id=model_id, input_per_mtok=float(m.group(1)), output_per_mtok=float(m.group(2)),157 context_length=ctx_len, max_output_tokens=max_out, features=features, meta={"section": section})158 else:159 low = texts[2].lower()160 if "per hour" in low or "per 1m characters" in low or "contact" in low:161 facts.claim(ref, "groq_pricing_note", texts[2][:120])162 facts.relate(ref, "available_through", provider, attributes={"provider_model_id": model_id, **features})163 if section == "preview":164 facts.claim(ref, "groq_status", "preview") # Groq's own availability tier; the model's `status` belongs to its lab165166167# ================================================================================================ Together AI168class TogetherPricingConnector(_PricingBase):169 name = "together_pricing"170 label = "Together AI — serverless inference pricing"171 description = "together.ai/pricing tables (serverless input / cached / output and batch prices per 1M tokens)."172 source_key = "together.ai"173 rate_per_min = 12174 expected_min_records = 10175 provider_key = "together-ai"176177 async def discover(self, ctx: RunContext) -> list[Target]:178 return [Target(url="https://www.together.ai/pricing", doc_type="pricing", key="together_pricing", min_bytes=50000, priority=1)]179180 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:181 facts = Facts()182 provider = provider_ref(self.provider_key)183 facts.entities.append(provider)184 facts.document_entity = provider185 html = parsed.html186 if not html:187 return facts188 rows: dict[str, dict[str, Any]] = {}189 for table in html.css("table"):190 headers = [th.text(strip=True).lower() for th in table.css("thead th")]191 if not headers or headers[0] != "model" or "input" not in headers:192 continue193 is_batch = "batch" in (table.parent.text(separator=" ", strip=True).lower()[:400] if table.parent else "") and "cached" not in table.text().lower()194 for tr in table.css("tbody tr"):195 tds = tr.css("td")196 if len(tds) < 3:197 continue198 a = tds[0].css_first("a[href*='/models/']")199 slug = a.attributes.get("href", "").rstrip("/").rsplit("/", 1)[-1] if a else None200 name = tds[0].text(strip=True)201 if not name:202 continue203 key = slug or name.lower()204 entry = rows.setdefault(key, {"name": name, "slug": slug, "input": None, "cached": None, "output": None, "batch_in": None, "batch_out": None})205 in_txt = tds[1].text(separator=" ", strip=True)206 out_txt = tds[2].text(separator=" ", strip=True)207 cached = None208 if "(cached)" in in_txt.lower():209 parts = [p for p in MONEY.findall(in_txt)]210 if len(parts) >= 2:211 cached = float(parts[1])212 if is_batch or (entry["input"] is not None and cached is None and entry["cached"] is not None):213 entry["batch_in"], entry["batch_out"] = _money(in_txt), _money(out_txt)214 else:215 entry["input"], entry["output"], entry["cached"] = _money(in_txt), _money(out_txt), cached216 for key, e in rows.items():217 if e["input"] is None and e["output"] is None:218 continue219 ids = {"together_ai_model_slug": e["slug"]} if e["slug"] else {}220 org = _org_for_id(facts, e["slug"], e["name"].split(" ")[0])221 ref = self._model(facts, e["name"], ids=ids, org=org, aliases=[e["slug"] or ""])222 facts.price(model=ref, provider=provider, provider_model_id=e["slug"], input_per_mtok=e["input"], output_per_mtok=e["output"],223 cached_input_per_mtok=e["cached"], batch_input_per_mtok=e["batch_in"], batch_output_per_mtok=e["batch_out"],224 features={"serverless": True}, source_url=f"https://www.together.ai/models/{e['slug']}" if e["slug"] else None)225 facts.document_title = html.title226 return facts227228229# ================================================================================================ Fireworks AI230class FireworksPricingConnector(_PricingBase):231 name = "fireworks_pricing"232 label = "Fireworks AI — serverless per-token pricing"233 description = "docs.fireworks.ai/serverless/pricing (Markdown): Standard and Priority input / cached / output prices per 1M tokens."234 source_key = "fireworks.ai"235 rate_per_min = 12236 expected_min_records = 10237 provider_key = "fireworks-ai"238239 async def discover(self, ctx: RunContext) -> list[Target]:240 return [Target(url="https://docs.fireworks.ai/serverless/pricing.md", doc_type="pricing", key="fireworks_serverless", min_bytes=2000, priority=1,241 accept="text/markdown, text/plain;q=0.9, */*;q=0.5", meta={"content_type": "text/markdown"})]242243 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:244 facts = Facts()245 provider = provider_ref(self.provider_key)246 facts.entities.append(provider)247 facts.document_entity = provider248 md = parsed.markdown249 if not md:250 return facts251 base_names: dict[str, str] = {}252 refs: dict[str, EntityRef] = {}253 for table in md.tables:254 headers = [h.strip().lower() for h in table["headers"]]255 if not headers or headers[0] != "model" or "standard" not in headers:256 continue257 i_std, i_pri = headers.index("standard"), headers.index("priority") if "priority" in headers else None258 i_res = next((i for i, h in enumerate(headers) if "reserved" in h), None)259 for row in table["rows"]:260 if len(row) <= i_std:261 continue262 label, link = _md_link(row[0])263 if not label:264 continue265 slug = None266 if link:267 m = re.search(r"app\.fireworks\.ai/models/([\w.-]+/[\w.-]+)", link)268 slug = m.group(1) if m else None269 std = [float(x) for x in MONEY.findall(row[i_std].replace("\\$", "$"))]270 if len(std) < 2:271 continue272 inp, cached, out = (std[0], std[1], std[2]) if len(std) >= 3 else (std[0], None, std[1])273 base_key = slug or label.lower()274 base_name = base_names.setdefault(base_key, label)275 variant = label[len(base_name):].strip().lower().replace(" ", "-") if label.startswith(base_name) and label != base_name else None276 ref = refs.get(base_key)277 if ref is None:278 org = _org_for_id(facts, slug, base_name.split(" ")[0])279 # canonical scheme `fireworks_model_id`; the historical hyphenated scheme is kept as a second identifier so existing rows keep resolving280 ids = {"fireworks_model_id": slug, "fireworks-ai_model_id": slug} if slug else {}281 ref = refs[base_key] = self._model(facts, base_name, ids=ids, org=org, aliases=[slug or "", slug.split("/")[-1] if slug else ""])282 features: dict[str, Any] = {"serving_path": variant or "standard"}283 if i_pri is not None and len(row) > i_pri:284 pri = [float(x) for x in MONEY.findall(row[i_pri].replace("\\$", "$"))]285 if len(pri) >= 2:286 features["priority"] = {"input_per_mtok": pri[0], "cached_input_per_mtok": pri[1] if len(pri) >= 3 else None, "output_per_mtok": pri[-1]}287 if i_res is not None and len(row) > i_res and "✓" in row[i_res]:288 features["reserved_throughput"] = True289 pmid = f"{slug}:{variant}" if slug and variant else slug or (f"{label.lower()}" if variant else None)290 facts.price(model=ref, provider=provider, provider_model_id=pmid, input_per_mtok=inp, output_per_mtok=out, cached_input_per_mtok=cached,291 features=features, source_url=link or None, meta={"row_label": label})292 facts.claim(provider, "batch_discount", "50% of serverless pricing") if "50% of serverless" in md.body else None293 facts.document_title = next((t for lvl, t in md.headings if lvl == 1), "Fireworks serverless pricing")294 return facts295296297def _md_link(cell: str) -> tuple[str | None, str | None]:298 m = re.match(r"\s*\[([^\]]+)\]\(([^)]+)\)", cell or "")299 if m:300 return m.group(1).strip(), m.group(2).strip()301 text = re.sub(r"[*_`]", "", cell or "").strip()302 return (text or None), None303304305CONNECTORS = [GroqPricingConnector, TogetherPricingConnector, FireworksPricingConnector]306