HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Hugging Face Hub — direct HTML only (no `/api/` endpoints): the hub pages embed their data as JSON in `data-props` attributes.23Sources (tier 2 for hub metadata — model-card statements are the authors' own):4 * listing pages https://huggingface.co/models?author=<org>&sort=downloads&p=<n> → `ModelList.initialValues.models` (30 per page:5 id, downloads, likes, pipeline_tag, lastModified, gated, numParameters). The organization page itself only embeds6 its 10 most recent models, so the author listing is used for discovery (configurable `models_per_org`, default 20).7 * model pages https://huggingface.co/<org>/<model> → `ModelHeader.model` (cardData, config, safetensors,8 tags, license, gated, createdAt, lastModified, downloads, likes) + GGUF/safetensors file links.9 * raw model card https://huggingface.co/<org>/<model>/raw/main/README.md → YAML front matter (license, base_model, datasets,10 language, quantized_by…) and the card text for later LLM passes. Not fetched for gated repositories (401 without auth).11 * daily papers https://huggingface.co/papers → `DailyPapers.dailyPapers` → paper entities (arXiv id).1213Identity (ontology `aiatlas.ontology.models`): a repository is either the official checkpoint of a MODEL (`Qwen/Qwen3-8B` → model14"Qwen3-8B", family hint "Qwen3") or an ARTIFACT of one — quantisation (`bartowski/Qwen3.8-27B-GGUF`, `zai-org/GLM-5-FP8`), conversion15(`mlx-community/Kimi-K2.5-bf16`, ONNX/CoreML repacks) or packaging (a converter organisation re-uploading the same weights). Artifacts keep the16full repo id as name and point to their canonical model through `EntityRef.canonical` (the `base_model` tag when present, otherwise the17analysed base name with medium identity confidence); the writer materialises `canonical_id` / `artifact_of`. Effort labels, gating and18licences are canonical: `license` is the ontology key (raw slug in `license_raw`), gating is `access: gated|open` (not an openness value),19`weights_available: true` for every hub repository and `openness` derived from the ontology dimensions.2021Discovery is bounded: `max_targets` (default 1500) and `models_per_org`; at 20 requests/minute a 1 100-target run takes ~55 minutes22(`aia run huggingface --max-targets 700` stays under 40 minutes).23"""24from __future__ import annotations2526import math27import re28from datetime import UTC, datetime29from typing import Any30from urllib.parse import quote3132from aiatlas.connectors._identity import family_ref33from aiatlas.ontology.licenses import normalize_license34from aiatlas.ontology.models import CONVERTER_ORGS, PRECISION_FORMATS, QUANT_FORMATS, NameAnalysis, analyze_model_name35from aiatlas.ontology.openness import derive_openness, openness_dimensions36from aiatlas.ontology.taxonomy import normalize_modalities37from aiatlas.registry import org_by_hf, org_ref, organizations38from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext39from aiatlas.sdk.extract.dates import parse_datetime40from aiatlas.sdk.extract.numbers import parse_active_params, parse_param_count41from aiatlas.sdk.facts import EntityRef, Facts, Target42from aiatlas.sdk.fetch import FetchResult4344HF = "https://huggingface.co"45PER_PAGE = 3046QUANT_TAGS = ("gguf", "mlx", "awq", "gptq", "fp8", "exl2", "exl3", "bitsandbytes", "onnx", "compressed-tensors", "quantized")47GGUF_QUANT = re.compile(r"[-_.](IQ\d+_[A-Z0-9_]+|Q\d+_[A-Z0-9_]+|Q\d+|BF16|F16|F32|MXFP4)\.gguf$", re.IGNORECASE)48ORIGINAL_MODEL = re.compile(r"(?:Original|Base|Source) model:?\s*\[?(?:https?://huggingface\.co/)?([\w.-]+/[\w.-]+)", re.IGNORECASE)49BASE_MODEL_TAG = re.compile(r"^base_model:(?:(finetune|quantized|merge|adapter):)?([\w.-]+/[\w.-]+)$")50RELATION_BY_KIND = {"finetune": "fine_tuned_from", "quantized": "quantized_from", "merge": "merged_from", "adapter": "derived_from", None: "derived_from"}51NOISE_TAGS = {"endpoints_compatible", "text-generation-inference", "eval-results", "autotrain_compatible", "has_space", "conversational"}52# Organizations that re-publish other labs' weights (quantizations, mirrors). Their repos are artifacts and keep the full `org/name` as53# entity name so that `unsloth/Llama-3.1-8B-Instruct` never merges by alias into Meta's `Llama-3.1-8B-Instruct`.54REDISTRIBUTORS = {"bartowski", "unsloth", "mlx-community", "thebloke", "lmstudio-community", "quantfactory", "mradermacher", "nvidia-community",55 "ggml-org", "second-state", "turboderp", "casperhansen", "hugging-quants", "neuralmagic", "redhatai"} | CONVERTER_ORGS56# Model families and the hub organizations that publish them: `NousResearch/Meta-Llama-3.1-70B-Instruct` is a mirror of Meta's repo,57# not Meta's entity, so it keeps its full id as name; the canonical model of a `bartowski/Qwen3.8-27B-GGUF` artifact belongs to Qwen.58FAMILY_ORGS = {r"^(meta-)?llama": {"meta-llama"}, r"^qwen|^qwq|^qvq": {"qwen"}, r"^deepseek": {"deepseek-ai"}, r"^(mistral|mixtral|magistral|devstral|codestral|ministral|pixtral|voxtral)": {"mistralai"},59 r"^gemma|^paligemma|^shieldgemma|^medgemma": {"google"}, r"^phi-": {"microsoft"}, r"^glm|^chatglm|^cogview|^cogvideo": {"zai-org", "thudm"}, r"^kimi": {"moonshotai"},60 r"^minimax": {"minimaxai"}, r"^gpt-oss": {"openai"}, r"^whisper": {"openai"}, r"^granite": {"ibm-granite"}, r"^(nvidia-)?nemotron": {"nvidia"},61 r"^smollm|^smolvlm": {"huggingfacetb"}, r"^olmo|^molmo|^tulu": {"allenai"}, r"^(c4ai-)?command|^aya": {"coherelabs", "cohereforai"}, r"^flux": {"black-forest-labs"},62 r"^stable-diffusion|^sdxl|^sd3": {"stabilityai"}, r"^claude": {"anthropic"}, r"^grok": {"xai-org"}, r"^seed-|^bagel": {"bytedance-seed"}, r"^hunyuan": {"tencent"},63 r"^ernie": {"baidu"}, r"^jamba": {"ai21labs"}, r"^lfm": {"liquidai"}, r"^dbrx": {"databricks"}, r"^hermes": {"nousresearch"}}64_QUANT_TOKEN = re.compile(r"^(w\d+a\d+|q\d(_[a-z0-9]+)*|iq\d(_[a-z0-9]+)*|\d-?bit|int\d|fp\d|nvfp\d|mxfp\d|ud-q\d.*|bnb-\d+bit)$", re.IGNORECASE)65# pipeline tags without a "-to-" arrow → (input modalities, output modalities); `-to-` tags are split on the arrow66PIPELINE_MODALITIES: dict[str, tuple[list[str], list[str]]] = {67 "text-generation": (["text"], ["text"]), "text2text-generation": (["text"], ["text"]), "fill-mask": (["text"], ["text"]), "translation": (["text"], ["text"]),68 "summarization": (["text"], ["text"]), "question-answering": (["text"], ["text"]), "conversational": (["text"], ["text"]),69 "text-classification": (["text"], ["structured"]), "token-classification": (["text"], ["structured"]), "zero-shot-classification": (["text"], ["structured"]),70 "feature-extraction": (["text"], ["embedding"]), "sentence-similarity": (["text"], ["embedding"]), "image-feature-extraction": (["image"], ["embedding"]),71 "automatic-speech-recognition": (["audio"], ["text"]), "audio-classification": (["audio"], ["structured"]), "text-to-speech": (["text"], ["audio"]),72 "text-to-audio": (["text"], ["audio"]), "audio-to-audio": (["audio"], ["audio"]), "voice-activity-detection": (["audio"], ["structured"]),73 "image-classification": (["image"], ["structured"]), "object-detection": (["image"], ["structured"]), "image-segmentation": (["image"], ["image"]),74 "zero-shot-image-classification": (["image", "text"], ["structured"]), "zero-shot-object-detection": (["image", "text"], ["structured"]),75 "depth-estimation": (["image"], ["image"]), "image-to-image": (["image"], ["image"]), "image-to-text": (["image"], ["text"]), "text-to-image": (["text"], ["image"]),76 "text-to-video": (["text"], ["video"]), "image-to-video": (["image"], ["video"]), "video-classification": (["video"], ["structured"]), "video-text-to-text": (["video", "text"], ["text"]),77 "image-text-to-text": (["image", "text"], ["text"]), "audio-text-to-text": (["audio", "text"], ["text"]), "visual-question-answering": (["image", "text"], ["text"]),78 "document-question-answering": (["document", "text"], ["text"]), "text-to-3d": (["text"], ["3d"]), "image-to-3d": (["image"], ["3d"]), "robotics": (["image", "text"], ["action"]),79 "reinforcement-learning": ([], ["action"]), "any-to-any": ([], []), "text-ranking": (["text"], ["structured"]), "mask-generation": (["image"], ["image"]),80 "keypoint-detection": (["image"], ["structured"]), "unconditional-image-generation": ([], ["image"]), "tabular-classification": (["structured"], ["structured"]),81 "tabular-regression": (["structured"], ["structured"]), "time-series-forecasting": (["structured"], ["structured"]), "graph-ml": (["structured"], ["structured"]),82}838485class HuggingFaceConnector(BaseConnector):86 name = "huggingface"87 label = "Hugging Face Hub — models of the registry organizations, model cards, daily papers"88 description = "Hub listing pages, model pages (embedded JSON), raw model cards and the daily-papers page — direct HTML, no API."89 source_key = "huggingface.co"90 version = "2"91 parser_version = "2"92 interval_seconds = 6 * 360093 min_interval_seconds = 3 * 360094 max_interval_seconds = 2 * 8640095 rate_per_min = 20 # huggingface.co answers 429 at 30/min with 3 parallel fetches (observed 2026-09-11); 20/min × 2 is clean96 tier = 297 priority = 198 expected_min_records = 20099 concurrency = 2100101 def __init__(self, config: dict[str, Any] | None = None):102 super().__init__(config)103 self.config.setdefault("max_targets", 1500)104 self.config.setdefault("models_per_org", 20)105 self.config.setdefault("listing_sort", "downloads")106 self.config.setdefault("fetch_readme", True)107108 # ------------------------------------------------------------------------------------------ discovery109 def hf_orgs(self) -> list[str]:110 if self.config.get("orgs"):111 return list(self.config["orgs"])112 return sorted({o["hf_org"] for o in organizations().values() if o.get("hf_org")}, key=str.lower)113114 async def discover(self, ctx: RunContext) -> list[Target]:115 targets = [Target(url=f"{HF}/papers", doc_type="listing", key="papers", min_bytes=5000, priority=1)]116 per_org = int(self.config["models_per_org"])117 pages = max(1, math.ceil(per_org / PER_PAGE))118 sort = self.config["listing_sort"]119 for org in self.hf_orgs():120 for p in range(pages):121 targets.append(Target(url=f"{HF}/models?author={quote(org)}&sort={sort}&p={p}", doc_type="listing", key=f"listing:{org}:{p}",122 meta={"hf_org": org, "page": p}, min_bytes=5000, priority=1))123 return targets124125 # ------------------------------------------------------------------------------------------ extraction126 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:127 facts = Facts()128 key = target.key or ""129 if key == "papers" and parsed.html:130 self._papers(facts, parsed)131 elif key.startswith("listing:") and parsed.html:132 self._listing(facts, target, parsed)133 elif target.doc_type == "model_page" and parsed.html:134 self._model_page(facts, target, parsed)135 elif target.doc_type == "model_card" and parsed.markdown:136 self._model_card(facts, target, parsed)137 return facts138139 # ------------------------------------------------------------------------------------------ listing140 def _listing(self, facts: Facts, target: Target, parsed: Parsed) -> None:141 html = parsed.html142 assert html143 props = html.embedded_json.get("data-props:ModelList") or {}144 models = ((props.get("initialValues") or {}).get("models")) or []145 hf_org = target.meta.get("hf_org") or ""146 author_data = next((m.get("authorData") for m in models if m.get("authorData")), None)147 org = self._org_ref(facts, hf_org, author_data)148 per_org = int(self.config["models_per_org"])149 page = int(target.meta.get("page") or 0)150 budget = per_org - page * PER_PAGE151 for m in models[: max(0, budget)]:152 repo_id = m.get("id")153 if not repo_id or m.get("private"):154 continue155 ref = self._model_ref(facts, repo_id, org)156 facts.claim(ref, "hf_repo", repo_id)157 facts.claim(ref, "model_card_url", f"{HF}/{repo_id}")158 self._pipeline_claims(facts, ref, m.get("pipeline_tag"))159 facts.claim(ref, "metric.downloads", m.get("downloads"))160 facts.claim(ref, "metric.likes", m.get("likes"))161 facts.claim(ref, "last_modified", _iso(m.get("lastModified")))162 gated = m.get("gated")163 self._access_claims(facts, ref, gated)164 if isinstance(m.get("numParameters"), int) and m["numParameters"] > 0:165 facts.claim(ref, "parameter_count", m["numParameters"])166 facts.follow(f"{HF}/{repo_id}", doc_type="model_page", entity=ref, key=f"model:{repo_id}", min_bytes=5000,167 meta={"hf_repo": repo_id, "hf_org": hf_org, "gated": bool(gated), "num_parameters": m.get("numParameters")})168 facts.document_entity = org169 facts.document_title = f"Hugging Face models — {hf_org}"170171 # ------------------------------------------------------------------------------------------ model page172 def _model_page(self, facts: Facts, target: Target, parsed: Parsed) -> None:173 html = parsed.html174 assert html175 header = html.embedded_json.get("data-props:ModelHeader") or {}176 model = header.get("model") or (html.embedded_json.get("data-props:ModelTensorsParams") or {}).get("model") or {}177 repo_id = model.get("id") or target.meta.get("hf_repo")178 if not repo_id:179 return180 author = model.get("author") or repo_id.split("/")[0]181 org = self._org_ref(facts, author, header.get("author"))182 card = model.get("cardData") or {}183 config = model.get("config") or {}184 tags = [t for t in (model.get("tags") or []) if isinstance(t, str)]185 gated = model.get("gated")186 bases = self._base_models(card, tags)187 quant_format = next((q for q in QUANT_TAGS if q in tags and q != "quantized"), None)188 if not quant_format and repo_id.lower().endswith(("-gguf", "_gguf")):189 quant_format = "gguf"190 if not quant_format and author.lower() == "mlx-community":191 quant_format = "mlx"192 ref = self._model_ref(facts, repo_id, org, bases=bases, quant_format=quant_format)193 is_artifact = ref.entity_type == "artifact"194 facts.claim(ref, "hf_repo", repo_id)195 facts.claim(ref, "model_card_url", f"{HF}/{repo_id}")196 self._pipeline_claims(facts, ref, model.get("pipeline_tag") or card.get("pipeline_tag"))197 facts.claim(ref, "library_name", model.get("library_name") or card.get("library_name"))198 license_ = card.get("license") or next((t.split(":", 1)[1] for t in tags if t.startswith("license:")), None)199 if isinstance(license_, list):200 license_ = license_[0] if license_ else None201 license_key = self._license_claims(facts, ref, license_, card.get("license_name"))202 facts.claim(ref, "license_url", card.get("license_link"))203 self._access_claims(facts, ref, gated, license_key=license_key)204 facts.claim(ref, "release_date", _date(model.get("createdAt")))205 facts.claim(ref, "last_modified", _iso(model.get("lastModified")))206 facts.claim(ref, "metric.downloads", model.get("downloads"))207 facts.claim(ref, "metric.downloads_all_time", model.get("downloadsAllTime"))208 facts.claim(ref, "metric.likes", model.get("likes"))209 # parameters: the packaged size on artifacts, the safetensors count on official checkpoints (never on the canonical model of an artifact)210 st = model.get("safetensors") or {}211 total = st.get("total") if isinstance(st, dict) else None212 listed = target.meta.get("num_parameters") # the hub's own count from the listing (GGUF repos have no safetensors)213 params = total if isinstance(total, int) and total > 0 else listed if isinstance(listed, int) and listed > 0 else parse_param_count(repo_id.split("/")[-1])214 facts.claim(ref, "parameter_count", params)215 facts.claim(ref, "active_parameter_count", parse_active_params(repo_id.split("/")[-1]))216 if isinstance(st, dict) and st.get("parameters"):217 facts.claim(ref, "weights_dtype", sorted(st["parameters"]))218 if isinstance(st, dict) and isinstance(st.get("totalFileSize"), int):219 facts.claim(ref, "file_size_gb", round(st["totalFileSize"] / 1e9, 2), unit="GB")220 archs = config.get("architectures") if isinstance(config, dict) else None221 facts.claim(ref, "architecture", archs[0] if isinstance(archs, list) and archs else None)222 facts.claim(ref, "model_type", config.get("model_type") if isinstance(config, dict) else None)223 facts.claim(ref, "languages", _listify(card.get("language")))224 datasets = _listify(card.get("datasets"))225 facts.claim(ref, "datasets", datasets)226 for ds in datasets[:20]:227 if "/" in ds or re.fullmatch(r"[\w.-]+", ds):228 facts.relate(ref, "uses_dataset", facts.entity("dataset", ds, identifiers={"hf_dataset": ds}))229 clean_tags = [t for t in tags if ":" not in t and t not in NOISE_TAGS][:30]230 facts.claim(ref, "tags", clean_tags)231 for t in tags:232 if t.startswith("arxiv:"):233 paper = facts.entity("paper", f"arXiv:{t[6:]}", identifiers={"arxiv": t[6:]})234 facts.relate(ref, "described_by", paper)235 # quantization236 facts.claim(ref, "quant_format", quant_format)237 quants = sorted({m.group(1).upper() for href, _ in html.links for m in [GGUF_QUANT.search(href)] if m and "/blob/main/" in href})238 if not quants and quant_format == "mlx":239 m = re.search(r"-(\d)bit\b", repo_id, re.IGNORECASE)240 quants = [f"{m.group(1)}bit"] if m else []241 facts.claim(ref, "quantization", quants)242 if quant_format or (is_artifact and ref.artifact_kind == "quantization"):243 facts.claim(ref, "is_quantized", True)244 if is_artifact:245 facts.claim(ref, "artifact_kind", ref.artifact_kind)246 # base model relations (artifact → model, fine-tune → base…)247 for base_id, kind in bases:248 base = self._base_ref(facts, base_id)249 facts.relate(ref, RELATION_BY_KIND.get(kind, "derived_from"), base, attributes={"base_model_relation": kind} if kind else {})250 facts.claim(ref, "base_model", [b for b, _ in bases] or None)251 facts.claim(ref, "quantized_by", card.get("quantized_by"))252 providers = [p.get("provider") for p in (model.get("availableInferenceProviders") or []) if isinstance(p, dict) and p.get("provider")]253 facts.claim(ref, "hf_inference_providers", sorted(set(providers)))254 facts.document_entity = ref255 facts.document_title = repo_id256 if self.config.get("fetch_readme", True) and not gated and model.get("cardExists", True):257 facts.follow(f"{HF}/{repo_id}/raw/main/README.md", doc_type="model_card", entity=ref, key=f"card:{repo_id}", priority=3,258 meta={"hf_repo": repo_id, "hf_org": author, "content_type": "text/markdown", "llm_task": "model_passport"}, min_bytes=32)259260 # ------------------------------------------------------------------------------------------ raw model card261 def _model_card(self, facts: Facts, target: Target, parsed: Parsed) -> None:262 md = parsed.markdown263 assert md264 repo_id = target.meta.get("hf_repo")265 if not repo_id:266 m = re.search(r"huggingface\.co/([\w.-]+/[\w.-]+)/raw/", target.url)267 repo_id = m.group(1) if m else None268 if not repo_id:269 return270 fm = md.front_matter or {}271 bases = [(b, None) for b in _listify(fm.get("base_model")) if "/" in b]272 quantized = bool(fm.get("quantized_by")) or "gguf" in [t.lower() for t in _listify(fm.get("tags"))]273 if not bases:274 m = ORIGINAL_MODEL.search(md.body)275 if m and m.group(1).lower() != repo_id.lower():276 bases = [(m.group(1), "quantized")]277 quantized = True278 org = self._org_ref(facts, repo_id.split("/")[0], None)279 ref = target.entity or self._model_ref(facts, repo_id, org, bases=bases, quant_format="gguf" if quantized else None)280 if ref not in facts.entities:281 facts.entities.append(ref)282 lic = fm.get("license")283 self._license_claims(facts, ref, lic[0] if isinstance(lic, list) and lic else (lic if isinstance(lic, str) else None), fm.get("license_name"))284 self._pipeline_claims(facts, ref, fm.get("pipeline_tag"))285 facts.claim(ref, "library_name", fm.get("library_name"))286 facts.claim(ref, "languages", _listify(fm.get("language")))287 facts.claim(ref, "datasets", _listify(fm.get("datasets")))288 facts.claim(ref, "quantized_by", fm.get("quantized_by"))289 for base_id, _kind in bases:290 base = self._base_ref(facts, base_id)291 # the typed relation (finetune/quantized/merge/adapter) comes from the model page tags; the card only proves quantization292 if quantized:293 facts.relate(ref, "quantized_from", base)294 facts.claim(ref, "base_model", [b for b, _ in bases] or None)295 h1 = next((t for lvl, t in md.headings if lvl == 1), None)296 facts.document_title = h1 or repo_id297 facts.document_entity = ref298299 # ------------------------------------------------------------------------------------------ daily papers300 def _papers(self, facts: Facts, parsed: Parsed) -> None:301 html = parsed.html302 assert html303 props = html.embedded_json.get("data-props:DailyPapers") or {}304 for item in props.get("dailyPapers") or []:305 paper = item.get("paper") or {}306 arxiv_id = paper.get("id")307 title = (paper.get("title") or item.get("title") or "").strip()308 if not arxiv_id or not title:309 continue310 published = parse_datetime(paper.get("publishedAt") or item.get("publishedAt"))311 ref = facts.entity("paper", title[:300], identifiers={"arxiv": arxiv_id}, first_seen_hint=published)312 facts.claim(ref, "arxiv_id", arxiv_id)313 facts.claim(ref, "authors", [a.get("name") for a in paper.get("authors") or [] if isinstance(a, dict) and a.get("name")][:50])314 facts.claim(ref, "abstract", (paper.get("summary") or "").strip()[:4000] or None)315 facts.claim(ref, "published_at", published.isoformat(timespec="seconds") if published else None)316 facts.claim(ref, "pdf_url", f"https://arxiv.org/pdf/{arxiv_id}")317 facts.claim(ref, "official_url", f"https://arxiv.org/abs/{arxiv_id}")318 facts.claim(ref, "hf_paper_url", f"{HF}/papers/{arxiv_id}")319 facts.claim(ref, "metric.upvotes", paper.get("upvotes"))320 facts.claim(ref, "metric.hf_comments", item.get("numComments"))321 facts.claim(ref, "github_repo", _github_repo(paper.get("githubRepo")))322 if paper.get("githubStars") is not None:323 facts.claim(ref, "metric.github_stars", paper.get("githubStars"))324 facts.document_title = f"Hugging Face daily papers — {props.get('dateString') or ''}".strip()325326 # ------------------------------------------------------------------------------------------ helpers327 def _org_ref(self, facts: Facts, hf_org: str, author_data: dict[str, Any] | None) -> EntityRef:328 known = org_by_hf(hf_org)329 if known:330 ref = org_ref(known["key"])331 else:332 display = (author_data or {}).get("fullname") or hf_org333 kind = "company" if (author_data or {}).get("type") == "org" else "organization"334 ref = EntityRef(entity_type=kind, name=display, identifiers={"hf_org": hf_org}, aliases=[hf_org])335 facts.claim(ref, "hf_org", hf_org)336 for e in facts.entities:337 if e.entity_type == ref.entity_type and e.identifiers and e.identifiers == ref.identifiers:338 return e339 facts.entities.append(ref)340 return ref341342 def _model_ref(self, facts: Facts, repo_id: str, org: EntityRef, *, bases: list[tuple[str, str | None]] | None = None,343 quant_format: str | None = None) -> EntityRef:344 """Model or artifact EntityRef for a repository (one per repo inside a Facts)."""345 for e in facts.entities:346 if e.entity_type in ("model", "artifact") and e.identifiers.get("hf_repo") == repo_id:347 return e348 org_slug, _, repo_name = repo_id.partition("/")349 analysis = analyze_model_name(repo_id)350 kind = artifact_kind(repo_id, analysis, bases=bases or [], quant_format=quant_format)351 if kind:352 canonical = self._canonical_ref(facts, repo_id, analysis, bases or [])353 ref = facts.entity("artifact", repo_id, identifiers={"hf_repo": repo_id}, organization=org, artifact_kind=kind, canonical=canonical,354 identity_confidence="high" if any("/" in b for b, _ in (bases or [])) else "medium")355 facts.relate(ref, "published_by", org)356 return ref357 foreign = _foreign_family(org_slug, repo_name)358 name = repo_id if foreign else (repo_name or repo_id)359 # an artifact seen earlier in the document may already have created this model by name (`Qwen/Qwen3.8-27B-FP8` before360 # `Qwen/Qwen3.8-27B`): upgrade that ref in place instead of creating a second one361 existing = next((e for e in facts.entities if e.entity_type == "model" and not e.identifiers and e.name.lower() == name.lower()362 and (e.organization is None or e.organization.identifiers == org.identifiers)), None)363 if existing is not None and not foreign:364 existing.identifiers["hf_repo"] = repo_id365 existing.organization = org366 existing.aliases = [a for a in dict.fromkeys([*existing.aliases, repo_id]) if a != name]367 existing.family = existing.family or family_ref(repo_name, org)368 existing.identity_confidence = "high"369 facts.relate(org, "develops", existing)370 return existing371 ref = facts.entity("model", name, identifiers={"hf_repo": repo_id}, organization=org, aliases=[repo_id] if not foreign else [],372 family=family_ref(repo_name, org), identity_confidence="medium" if foreign else "high")373 facts.relate(org, "develops", ref)374 return ref375376 def _base_ref(self, facts: Facts, base_id: str) -> EntityRef:377 """The model behind a `base_model` tag: an official repository → model ref (no `develops` here; its own page states it)."""378 for e in facts.entities:379 if e.entity_type in ("model", "artifact") and e.identifiers.get("hf_repo") == base_id:380 return e381 base_org_slug, _, base_name = base_id.partition("/")382 base_org = self._org_ref(facts, base_org_slug, None)383 analysis = analyze_model_name(base_id)384 if artifact_kind(base_id, analysis, bases=[], quant_format=None):385 # a quantisation of a quantisation: the base is itself an artifact of the analysed model386 ref = facts.entity("artifact", base_id, identifiers={"hf_repo": base_id}, organization=base_org,387 artifact_kind=artifact_kind(base_id, analysis, bases=[], quant_format=None),388 canonical=self._canonical_ref(facts, base_id, analysis, []), identity_confidence="medium")389 else:390 foreign = _foreign_family(base_org_slug, base_name)391 ref = facts.entity("model", base_id if foreign else base_name, identifiers={"hf_repo": base_id}, organization=base_org,392 aliases=[base_id] if not foreign else [], family=family_ref(base_name, base_org), identity_confidence="high")393 facts.claim(ref, "hf_repo", base_id)394 return ref395396 def _canonical_ref(self, facts: Facts, repo_id: str, analysis: NameAnalysis, bases: list[tuple[str, str | None]]) -> EntityRef | None:397 """The model an artifact packages: the `base_model` repo when present, otherwise the analysed base name (medium confidence) —398 `nvidia/Gemma-4-31B-IT-NVFP4` → same-org model "Gemma-4-31B-IT". None when the base cannot be named confidently (no token to strip399 from the name, or a redistributor repo whose family is unknown): never an invented model."""400 base = next((b for b, k in bases if k == "quantized" and "/" in b), None) or next((b for b, _ in bases if "/" in b), None)401 if base and base.lower() != repo_id.lower():402 return self._base_ref(facts, base)403 org_slug, _, repo_name = repo_id.partition("/")404 name = canonical_name(repo_name)405 if not name or name.lower() == repo_name.lower() or not re.search(r"[a-z]", name, re.IGNORECASE):406 return None407 org: EntityRef | None = None408 family_orgs = _family_orgs(repo_name)409 if family_orgs and org_slug.lower() in family_orgs:410 org = self._org_ref(facts, org_slug, None)411 elif family_orgs:412 hf_org = next((o for o in sorted(family_orgs) if org_by_hf(o)), None)413 org = self._org_ref(facts, org_by_hf(hf_org)["hf_org"], None) if hf_org else None # type: ignore[index]414 elif org_slug.lower() not in REDISTRIBUTORS:415 org = self._org_ref(facts, org_slug, None) # the official developer's own quantisation / conversion → its own model416 if org is None:417 return None418 for e in facts.entities:419 if e.entity_type == "model" and e.name.lower() == name.lower() and (org is None or e.organization is None or e.organization.identifiers == org.identifiers):420 return e421 ref = facts.entity("model", name, organization=org, family=family_ref(name, org), identity_confidence="medium")422 return ref423424 @staticmethod425 def _pipeline_claims(facts: Facts, ref: EntityRef, tag: Any) -> None:426 if not isinstance(tag, str) or not tag:427 return428 facts.claim(ref, "pipeline_tag", tag)429 mods_in, mods_out = pipeline_modalities(tag)430 facts.claim(ref, "modalities_input", mods_in)431 facts.claim(ref, "modalities_output", mods_out)432 facts.claim(ref, "modalities", sorted(set(mods_in) | set(mods_out)))433434 @staticmethod435 def _license_claims(facts: Facts, ref: EntityRef, raw: str | None, license_name: Any = None) -> str | None:436 if not raw:437 return None438 key = normalize_license(raw)439 if (key is None or key == "Other") and isinstance(license_name, str) and license_name.strip():440 key = normalize_license(license_name) or key441 facts.claim(ref, "license_name", license_name.strip())442 facts.claim(ref, "license_raw", raw)443 facts.claim(ref, "license", key or raw)444 return key445446 @staticmethod447 def _access_claims(facts: Facts, ref: EntityRef, gated: Any, *, license_key: str | None = None) -> None:448 facts.claim(ref, "access", "gated" if gated else "open")449 if isinstance(gated, str) and gated:450 facts.claim(ref, "gated_mode", gated)451 facts.claim(ref, "weights_available", True)452 if license_key is not None: # the category needs the licence terms; weights alone only prove `weights_available`453 dims = openness_dimensions(weights_available=True, license_key=license_key)454 facts.claim(ref, "openness", derive_openness(dims, license_key=license_key))455456 @staticmethod457 def _base_models(card: dict[str, Any], tags: list[str]) -> list[tuple[str, str | None]]:458 kinds: dict[str, str | None] = {}459 for t in tags:460 m = BASE_MODEL_TAG.match(t)461 if m:462 kinds[m.group(2)] = m.group(1) or kinds.get(m.group(2))463 for b in _listify(card.get("base_model")):464 if "/" in b:465 kinds.setdefault(b, None)466 return list(kinds.items())467468469# ---------------------------------------------------------------------------------------------- identity helpers (pure)470def artifact_kind(repo_id: str, analysis: NameAnalysis | None = None, *, bases: list[tuple[str, str | None]], quant_format: str | None) -> str | None:471 """quantization | conversion | packaging for a repository that is not the official checkpoint of a model, else None."""472 a = analysis or analyze_model_name(repo_id)473 org_slug = repo_id.split("/")[0].lower() if "/" in repo_id else ""474 if a.is_quantized or quant_format or any(k == "quantized" for _, k in bases):475 return "quantization"476 if a.quant_formats or a.precision:477 return "conversion"478 if org_slug in REDISTRIBUTORS:479 return "packaging"480 return None481482483def canonical_name(repo_name: str) -> str:484 """Repository name without quantisation / precision / packaging tokens, original casing kept: 'Qwen3.8-27B-GGUF' → 'Qwen3.8-27B',485 'Llama-3.1-8B-Instruct-bnb-4bit' → 'Llama-3.1-8B-Instruct', 'Kimi-K2.5-bf16' → 'Kimi-K2.5'."""486 kept: list[str] = []487 for tok in repo_name.split("-"):488 low = tok.lower()489 if not tok or low in QUANT_FORMATS or low in PRECISION_FORMATS or _QUANT_TOKEN.match(low) or low == "ud" or re.match(r"^i?q\d", low):490 continue491 kept.append(tok)492 return "-".join(kept).strip("-") or repo_name493494495def pipeline_modalities(tag: str) -> tuple[list[str], list[str]]:496 """Hub pipeline tag → (input modalities, output modalities), canonical vocabulary."""497 low = tag.strip().lower()498 if low in PIPELINE_MODALITIES:499 i, o = PIPELINE_MODALITIES[low]500 return normalize_modalities(i), normalize_modalities(o)501 if "-to-" in low:502 left, _, right = low.partition("-to-")503 return normalize_modalities(left.split("-")), normalize_modalities(right.split("-"))504 return [], []505506507def _family_orgs(repo_name: str) -> set[str] | None:508 low = repo_name.lower()509 for pattern, orgs in FAMILY_ORGS.items():510 if re.search(pattern, low):511 return {o.lower() for o in orgs}512 return None513514515def _foreign_family(org_slug: str, repo_name: str) -> bool:516 """True when the repo name belongs to a well-known model family published by another hub organization (mirror / re-upload)."""517 orgs = _family_orgs(repo_name)518 return org_slug.lower() not in orgs if orgs else False519520521def _listify(v: Any) -> list[str]:522 if v is None:523 return []524 if isinstance(v, str):525 return [v] if v.strip() else []526 if isinstance(v, list):527 return [str(x) for x in v if isinstance(x, (str, int, float)) and str(x).strip()]528 return []529530531def _iso(v: Any) -> str | None:532 dt = parse_datetime(v) if isinstance(v, str) else None533 return dt.astimezone(UTC).isoformat(timespec="seconds") if dt else None534535536def _date(v: Any) -> str | None:537 dt = parse_datetime(v) if isinstance(v, str) else None538 return dt.date().isoformat() if dt else None539540541def _github_repo(v: Any) -> str | None:542 if not isinstance(v, str):543 return None544 m = re.search(r"github\.com/([\w.-]+/[\w.-]+)", v)545 return m.group(1).removesuffix(".git") if m else None546547548def _now() -> datetime:549 return datetime.now(UTC)550551552CONNECTORS = [HuggingFaceConnector]553