1"""Hugging Face Hub — direct HTML only (no `/api/` endpoints): the hub pages embed their data as JSON in `data-props` attributes.23Sources (tier 2 for hub metadata — model-card statements are the authors' own):4 * listing pages https://huggingface.co/models?author=<org>&sort=downloads&p=<n> → `ModelList.initialValues.models` (30 per page:5 id, downloads, likes, pipeline_tag, lastModified, gated, numParameters). The organization page itself only embeds6 its 10 most recent models, so the author listing is used for discovery (configurable `models_per_org`, default 20).7 * model pages https://huggingface.co/<org>/<model> → `ModelHeader.model` (cardData, config, safetensors,8 tags, license, gated, createdAt, lastModified, downloads, likes) + GGUF/safetensors file links.9 * raw model card https://huggingface.co/<org>/<model>/raw/main/README.md → YAML front matter (license, base_model, datasets,10 language, quantized_by…) and the card text for later LLM passes. Not fetched for gated repositories (401 without auth).11 * daily papers https://huggingface.co/papers → `DailyPapers.dailyPapers` → paper entities (arXiv id).1213Discovery is bounded: `max_targets` (default 1500) and `models_per_org`; at 20 requests/minute a 1 100-target run takes ~55 minutes14(`aia run huggingface --max-targets 700` stays under 40 minutes).15"""16from __future__ import annotations1718import math19import re20from datetime import UTC, datetime21from typing import Any22from urllib.parse import quote2324from aiatlas.registry import org_by_hf, org_ref, organizations25from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext26from aiatlas.sdk.extract.dates import parse_datetime27from aiatlas.sdk.extract.numbers import parse_active_params, parse_param_count28from aiatlas.sdk.facts import EntityRef, Facts, Target29from aiatlas.sdk.fetch import FetchResult3031HF = "https://huggingface.co"32PER_PAGE = 3033QUANT_FORMATS = ("gguf", "mlx", "awq", "gptq", "fp8", "exl2", "exl3", "bitsandbytes", "onnx")34GGUF_QUANT = re.compile(r"[-_.](IQ\d+_[A-Z0-9_]+|Q\d+_[A-Z0-9_]+|Q\d+|BF16|F16|F32|MXFP4)\.gguf$", re.IGNORECASE)35ORIGINAL_MODEL = re.compile(r"(?:Original|Base|Source) model:?\s*\[?(?:https?://huggingface\.co/)?([\w.-]+/[\w.-]+)", re.IGNORECASE)36BASE_MODEL_TAG = re.compile(r"^base_model:(?:(finetune|quantized|merge|adapter):)?([\w.-]+/[\w.-]+)$")37RELATION_BY_KIND = {"finetune": "fine_tuned_from", "quantized": "quantized_from", "merge": "merged_from", "adapter": "derived_from", None: "derived_from"}38NOISE_TAGS = {"endpoints_compatible", "text-generation-inference", "eval-results", "autotrain_compatible", "has_space", "conversational"}39# Organizations that re-publish other labs' weights (quantizations, mirrors). Their repos keep the full `org/name` as entity name so that40# `unsloth/Llama-3.1-8B-Instruct` never merges by alias into Meta's `Llama-3.1-8B-Instruct` (the resolver merges on normalized name).41REDISTRIBUTORS = {"bartowski", "unsloth", "mlx-community", "thebloke", "lmstudio-community", "quantfactory", "mradermacher", "nvidia-community",42 "ggml-org", "second-state", "turboderp", "casperhansen", "hugging-quants", "neuralmagic", "redhatai"}43# Model families and the hub organizations that publish them: `NousResearch/Meta-Llama-3.1-70B-Instruct` is a mirror of Meta's repo,44# not Meta's entity, so it keeps its full id as name.45FAMILY_ORGS = {r"^(meta-)?llama": {"meta-llama"}, r"^qwen|^qwq|^qvq": {"qwen"}, r"^deepseek": {"deepseek-ai"}, r"^(mistral|mixtral|magistral|devstral|codestral|ministral|pixtral|voxtral)": {"mistralai"},46 r"^gemma|^paligemma|^shieldgemma|^medgemma": {"google"}, r"^phi-": {"microsoft"}, r"^glm|^chatglm|^cogview|^cogvideo": {"zai-org", "thudm"}, r"^kimi": {"moonshotai"},47 r"^minimax": {"minimaxai"}, r"^gpt-oss": {"openai"}, r"^whisper": {"openai"}, r"^granite": {"ibm-granite"}, r"^(nvidia-)?nemotron": {"nvidia"},48 r"^smollm|^smolvlm": {"huggingfacetb"}, r"^olmo|^molmo|^tulu": {"allenai"}, r"^(c4ai-)?command|^aya": {"coherelabs", "cohereforai"}, r"^flux": {"black-forest-labs"},49 r"^stable-diffusion|^sdxl|^sd3": {"stabilityai"}, r"^claude": {"anthropic"}, r"^grok": {"xai-org"}, r"^seed-|^bagel": {"bytedance-seed"}, r"^hunyuan": {"tencent"},50 r"^ernie": {"baidu"}, r"^jamba": {"ai21labs"}, r"^lfm": {"liquidai"}, r"^dbrx": {"databricks"}, r"^hermes": {"nousresearch"}}51DERIVED_SUFFIX = re.compile(r"[-_.](gguf|mlx|awq|gptq|exl2|exl3|fp8|nvfp4|int4|int8|bnb[-_]4bit|4bit|8bit|mxfp4)(?:[-_.][\w.]+)?$", re.IGNORECASE)525354class HuggingFaceConnector(BaseConnector):55 name = "huggingface"56 label = "Hugging Face Hub — models of the registry organizations, model cards, daily papers"57 description = "Hub listing pages, model pages (embedded JSON), raw model cards and the daily-papers page — direct HTML, no API."58 source_key = "huggingface.co"59 version = "1"60 parser_version = "1"61 interval_seconds = 6 * 360062 min_interval_seconds = 3 * 360063 max_interval_seconds = 2 * 8640064 rate_per_min = 20 # huggingface.co answers 429 at 30/min with 3 parallel fetches (observed 2026-09-11); 20/min × 2 is clean65 tier = 266 priority = 167 expected_min_records = 20068 concurrency = 26970 def __init__(self, config: dict[str, Any] | None = None):71 super().__init__(config)72 self.config.setdefault("max_targets", 1500)73 self.config.setdefault("models_per_org", 20)74 self.config.setdefault("listing_sort", "downloads")75 self.config.setdefault("fetch_readme", True)7677 # ------------------------------------------------------------------------------------------ discovery78 def hf_orgs(self) -> list[str]:79 if self.config.get("orgs"):80 return list(self.config["orgs"])81 return sorted({o["hf_org"] for o in organizations().values() if o.get("hf_org")}, key=str.lower)8283 async def discover(self, ctx: RunContext) -> list[Target]:84 targets = [Target(url=f"{HF}/papers", doc_type="listing", key="papers", min_bytes=5000, priority=1)]85 per_org = int(self.config["models_per_org"])86 pages = max(1, math.ceil(per_org / PER_PAGE))87 sort = self.config["listing_sort"]88 for org in self.hf_orgs():89 for p in range(pages):90 targets.append(Target(url=f"{HF}/models?author={quote(org)}&sort={sort}&p={p}", doc_type="listing", key=f"listing:{org}:{p}",91 meta={"hf_org": org, "page": p}, min_bytes=5000, priority=1))92 return targets9394 # ------------------------------------------------------------------------------------------ extraction95 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:96 facts = Facts()97 key = target.key or ""98 if key == "papers" and parsed.html:99 self._papers(facts, parsed)100 elif key.startswith("listing:") and parsed.html:101 self._listing(facts, target, parsed)102 elif target.doc_type == "model_page" and parsed.html:103 self._model_page(facts, target, parsed)104 elif target.doc_type == "model_card" and parsed.markdown:105 self._model_card(facts, target, parsed)106 return facts107108 # ------------------------------------------------------------------------------------------ listing109 def _listing(self, facts: Facts, target: Target, parsed: Parsed) -> None:110 html = parsed.html111 assert html112 props = html.embedded_json.get("data-props:ModelList") or {}113 models = ((props.get("initialValues") or {}).get("models")) or []114 hf_org = target.meta.get("hf_org") or ""115 author_data = next((m.get("authorData") for m in models if m.get("authorData")), None)116 org = self._org_ref(facts, hf_org, author_data)117 per_org = int(self.config["models_per_org"])118 page = int(target.meta.get("page") or 0)119 budget = per_org - page * PER_PAGE120 for m in models[: max(0, budget)]:121 repo_id = m.get("id")122 if not repo_id or m.get("private"):123 continue124 ref = self._model_ref(facts, repo_id, org)125 facts.claim(ref, "hf_repo", repo_id)126 facts.claim(ref, "model_card_url", f"{HF}/{repo_id}")127 facts.claim(ref, "pipeline_tag", m.get("pipeline_tag"))128 facts.claim(ref, "metric.downloads", m.get("downloads"))129 facts.claim(ref, "metric.likes", m.get("likes"))130 facts.claim(ref, "last_modified", _iso(m.get("lastModified")))131 gated = m.get("gated")132 facts.claim(ref, "openness", "restricted" if gated else "open-weights")133 if isinstance(m.get("numParameters"), int) and m["numParameters"] > 0:134 facts.claim(ref, "parameter_count", m["numParameters"])135 facts.follow(f"{HF}/{repo_id}", doc_type="model_page", entity=ref, key=f"model:{repo_id}", min_bytes=5000,136 meta={"hf_repo": repo_id, "hf_org": hf_org, "gated": bool(gated)})137 facts.document_entity = org138 facts.document_title = f"Hugging Face models — {hf_org}"139140 # ------------------------------------------------------------------------------------------ model page141 def _model_page(self, facts: Facts, target: Target, parsed: Parsed) -> None:142 html = parsed.html143 assert html144 header = html.embedded_json.get("data-props:ModelHeader") or {}145 model = header.get("model") or (html.embedded_json.get("data-props:ModelTensorsParams") or {}).get("model") or {}146 repo_id = model.get("id") or target.meta.get("hf_repo")147 if not repo_id:148 return149 author = model.get("author") or repo_id.split("/")[0]150 org = self._org_ref(facts, author, header.get("author"))151 ref = self._model_ref(facts, repo_id, org)152 card = model.get("cardData") or {}153 config = model.get("config") or {}154 tags = [t for t in (model.get("tags") or []) if isinstance(t, str)]155 gated = model.get("gated")156 facts.claim(ref, "hf_repo", repo_id)157 facts.claim(ref, "model_card_url", f"{HF}/{repo_id}")158 facts.claim(ref, "pipeline_tag", model.get("pipeline_tag") or card.get("pipeline_tag"))159 facts.claim(ref, "library_name", model.get("library_name") or card.get("library_name"))160 license_ = card.get("license") or next((t.split(":", 1)[1] for t in tags if t.startswith("license:")), None)161 if isinstance(license_, list):162 license_ = license_[0] if license_ else None163 facts.claim(ref, "license", license_)164 facts.claim(ref, "license_url", card.get("license_link"))165 facts.claim(ref, "openness", "restricted" if gated else "open-weights")166 facts.claim(ref, "gated", str(gated) if gated else False)167 facts.claim(ref, "release_date", _date(model.get("createdAt")))168 facts.claim(ref, "last_modified", _iso(model.get("lastModified")))169 facts.claim(ref, "metric.downloads", model.get("downloads"))170 facts.claim(ref, "metric.downloads_all_time", model.get("downloadsAllTime"))171 facts.claim(ref, "metric.likes", model.get("likes"))172 # parameters173 st = model.get("safetensors") or {}174 total = st.get("total") if isinstance(st, dict) else None175 params = total if isinstance(total, int) and total > 0 else parse_param_count(repo_id.split("/")[-1])176 facts.claim(ref, "parameter_count", params)177 facts.claim(ref, "active_parameter_count", parse_active_params(repo_id.split("/")[-1]))178 if isinstance(st, dict) and st.get("parameters"):179 facts.claim(ref, "weights_dtype", sorted(st["parameters"]))180 if isinstance(st, dict) and isinstance(st.get("totalFileSize"), int):181 facts.claim(ref, "file_size_gb", round(st["totalFileSize"] / 1e9, 2), unit="GB")182 archs = config.get("architectures") if isinstance(config, dict) else None183 facts.claim(ref, "architecture", archs[0] if isinstance(archs, list) and archs else None)184 facts.claim(ref, "model_type", config.get("model_type") if isinstance(config, dict) else None)185 facts.claim(ref, "languages", _listify(card.get("language")))186 datasets = _listify(card.get("datasets"))187 facts.claim(ref, "datasets", datasets)188 for ds in datasets[:20]:189 if "/" in ds or re.fullmatch(r"[\w.-]+", ds):190 facts.relate(ref, "uses_dataset", facts.entity("dataset", ds, identifiers={"hf_dataset": ds}))191 clean_tags = [t for t in tags if ":" not in t and t not in NOISE_TAGS][:30]192 facts.claim(ref, "tags", clean_tags)193 for t in tags:194 if t.startswith("arxiv:"):195 paper = facts.entity("paper", f"arXiv:{t[6:]}", identifiers={"arxiv": t[6:]})196 facts.relate(ref, "described_by", paper)197 # quantization198 quant_format = next((q for q in QUANT_FORMATS if q in tags), None)199 if not quant_format and repo_id.lower().endswith(("-gguf", "_gguf")):200 quant_format = "gguf"201 if not quant_format and author.lower() == "mlx-community":202 quant_format = "mlx"203 facts.claim(ref, "quant_format", quant_format)204 quants = sorted({m.group(1).upper() for href, _ in html.links for m in [GGUF_QUANT.search(href)] if m and "/blob/main/" in href})205 if not quants and quant_format == "mlx":206 m = re.search(r"-(\d)bit\b", repo_id, re.IGNORECASE)207 quants = [f"{m.group(1)}bit"] if m else []208 facts.claim(ref, "quantization", quants)209 if quant_format:210 facts.claim(ref, "is_quantized", True)211 # base model relations212 for base_id, kind in self._base_models(card, tags):213 base = self._model_ref(facts, base_id, self._org_ref(facts, base_id.split("/")[0], None), develops=False)214 facts.claim(base, "hf_repo", base_id)215 facts.relate(ref, RELATION_BY_KIND.get(kind, "derived_from"), base, attributes={"base_model_relation": kind} if kind else {})216 facts.claim(ref, "base_model", [b for b, _ in self._base_models(card, tags)] or None)217 facts.claim(ref, "quantized_by", card.get("quantized_by"))218 providers = [p.get("provider") for p in (model.get("availableInferenceProviders") or []) if isinstance(p, dict) and p.get("provider")]219 facts.claim(ref, "hf_inference_providers", sorted(set(providers)))220 facts.document_entity = ref221 facts.document_title = repo_id222 if self.config.get("fetch_readme", True) and not gated and model.get("cardExists", True):223 facts.follow(f"{HF}/{repo_id}/raw/main/README.md", doc_type="model_card", entity=ref, key=f"card:{repo_id}", priority=3,224 meta={"hf_repo": repo_id, "hf_org": author, "content_type": "text/markdown", "llm_task": "model_passport"}, min_bytes=32)225226 # ------------------------------------------------------------------------------------------ raw model card227 def _model_card(self, facts: Facts, target: Target, parsed: Parsed) -> None:228 md = parsed.markdown229 assert md230 repo_id = target.meta.get("hf_repo")231 if not repo_id:232 m = re.search(r"huggingface\.co/([\w.-]+/[\w.-]+)/raw/", target.url)233 repo_id = m.group(1) if m else None234 if not repo_id:235 return236 org = self._org_ref(facts, repo_id.split("/")[0], None)237 ref = target.entity or self._model_ref(facts, repo_id, org)238 if ref not in facts.entities:239 facts.entities.append(ref)240 fm = md.front_matter or {}241 lic = fm.get("license")242 facts.claim(ref, "license", lic[0] if isinstance(lic, list) and lic else (lic if isinstance(lic, str) else None))243 facts.claim(ref, "license_name", fm.get("license_name"))244 facts.claim(ref, "pipeline_tag", fm.get("pipeline_tag"))245 facts.claim(ref, "library_name", fm.get("library_name"))246 facts.claim(ref, "languages", _listify(fm.get("language")))247 facts.claim(ref, "datasets", _listify(fm.get("datasets")))248 facts.claim(ref, "quantized_by", fm.get("quantized_by"))249 bases = [b for b in _listify(fm.get("base_model")) if "/" in b]250 quantized = bool(fm.get("quantized_by")) or "gguf" in [t.lower() for t in _listify(fm.get("tags"))]251 if not bases:252 m = ORIGINAL_MODEL.search(md.body)253 if m and m.group(1).lower() != repo_id.lower():254 bases = [m.group(1)]255 quantized = True256 for base_id in bases:257 base = self._model_ref(facts, base_id, self._org_ref(facts, base_id.split("/")[0], None), develops=False)258 facts.claim(base, "hf_repo", base_id)259 # the typed relation (finetune/quantized/merge/adapter) comes from the model page tags; the card only proves quantization260 if quantized:261 facts.relate(ref, "quantized_from", base)262 facts.claim(ref, "base_model", bases or None)263 h1 = next((t for lvl, t in md.headings if lvl == 1), None)264 facts.document_title = h1 or repo_id265 facts.document_entity = ref266267 # ------------------------------------------------------------------------------------------ daily papers268 def _papers(self, facts: Facts, parsed: Parsed) -> None:269 html = parsed.html270 assert html271 props = html.embedded_json.get("data-props:DailyPapers") or {}272 for item in props.get("dailyPapers") or []:273 paper = item.get("paper") or {}274 arxiv_id = paper.get("id")275 title = (paper.get("title") or item.get("title") or "").strip()276 if not arxiv_id or not title:277 continue278 published = parse_datetime(paper.get("publishedAt") or item.get("publishedAt"))279 ref = facts.entity("paper", title[:300], identifiers={"arxiv": arxiv_id}, first_seen_hint=published)280 facts.claim(ref, "arxiv_id", arxiv_id)281 facts.claim(ref, "authors", [a.get("name") for a in paper.get("authors") or [] if isinstance(a, dict) and a.get("name")][:50])282 facts.claim(ref, "abstract", (paper.get("summary") or "").strip()[:4000] or None)283 facts.claim(ref, "published_at", published.isoformat(timespec="seconds") if published else None)284 facts.claim(ref, "pdf_url", f"https://arxiv.org/pdf/{arxiv_id}")285 facts.claim(ref, "official_url", f"https://arxiv.org/abs/{arxiv_id}")286 facts.claim(ref, "hf_paper_url", f"{HF}/papers/{arxiv_id}")287 facts.claim(ref, "metric.upvotes", paper.get("upvotes"))288 facts.claim(ref, "metric.hf_comments", item.get("numComments"))289 facts.claim(ref, "github_repo", _github_repo(paper.get("githubRepo")))290 if paper.get("githubStars") is not None:291 facts.claim(ref, "metric.github_stars", paper.get("githubStars"))292 facts.document_title = f"Hugging Face daily papers — {props.get('dateString') or ''}".strip()293294 # ------------------------------------------------------------------------------------------ helpers295 def _org_ref(self, facts: Facts, hf_org: str, author_data: dict[str, Any] | None) -> EntityRef:296 known = org_by_hf(hf_org)297 if known:298 ref = org_ref(known["key"])299 else:300 display = (author_data or {}).get("fullname") or hf_org301 kind = "company" if (author_data or {}).get("type") == "org" else "organization"302 ref = EntityRef(entity_type=kind, name=display, identifiers={"hf_org": hf_org}, aliases=[hf_org])303 facts.claim(ref, "hf_org", hf_org)304 if ref not in facts.entities:305 facts.entities.append(ref)306 return ref307308 def _model_ref(self, facts: Facts, repo_id: str, org: EntityRef, *, develops: bool = True) -> EntityRef:309 for e in facts.entities:310 if e.entity_type == "model" and e.identifiers.get("hf_repo") == repo_id:311 return e312 org_slug, _, repo_name = repo_id.partition("/")313 derived = org_slug.lower() in REDISTRIBUTORS or bool(DERIVED_SUFFIX.search(repo_name)) or _foreign_family(org_slug, repo_name)314 name = repo_id if derived else (repo_name or repo_id)315 ref = facts.entity("model", name, identifiers={"hf_repo": repo_id}, organization=org, aliases=[repo_id] if not derived else [])316 if develops:317 facts.relate(org, "develops", ref)318 return ref319320 @staticmethod321 def _base_models(card: dict[str, Any], tags: list[str]) -> list[tuple[str, str | None]]:322 kinds: dict[str, str | None] = {}323 for t in tags:324 m = BASE_MODEL_TAG.match(t)325 if m:326 kinds[m.group(2)] = m.group(1) or kinds.get(m.group(2))327 for b in _listify(card.get("base_model")):328 if "/" in b:329 kinds.setdefault(b, None)330 return list(kinds.items())331332333def _foreign_family(org_slug: str, repo_name: str) -> bool:334 """True when the repo name belongs to a well-known model family published by another hub organization (mirror / re-upload)."""335 low = repo_name.lower()336 for pattern, orgs in FAMILY_ORGS.items():337 if re.search(pattern, low):338 return org_slug.lower() not in orgs339 return False340341342def _listify(v: Any) -> list[str]:343 if v is None:344 return []345 if isinstance(v, str):346 return [v] if v.strip() else []347 if isinstance(v, list):348 return [str(x) for x in v if isinstance(x, (str, int, float)) and str(x).strip()]349 return []350351352def _iso(v: Any) -> str | None:353 dt = parse_datetime(v) if isinstance(v, str) else None354 return dt.astimezone(UTC).isoformat(timespec="seconds") if dt else None355356357def _date(v: Any) -> str | None:358 dt = parse_datetime(v) if isinstance(v, str) else None359 return dt.date().isoformat() if dt else None360361362def _github_repo(v: Any) -> str | None:363 if not isinstance(v, str):364 return None365 m = re.search(r"github\.com/([\w.-]+/[\w.-]+)", v)366 return m.group(1).removesuffix(".git") if m else None367368369def _now() -> datetime:370 return datetime.now(UTC)371372373CONNECTORS = [HuggingFaceConnector]374