HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""GitHub — seed list of AI repositories (registry/repositories.yaml), server-rendered pages only (no API).23Per repository (tier 2):4 * https://github.com/<owner>/<name> → the repository overview embeds `react-app.embeddedData` JSON (`payload.sidebarAbout`:5 description, website, topics, stargazerCount, forksCount, license, release count; `repo.createdAt`).6 CSS fallbacks: `#repo-stars-counter-star[title]`, `#repo-network-counter[title]`,7 `[data-content$=" license"]`, `a[href^="/topics/"]`. The language bar is client-loaded → no `language` claim.8 * https://github.com/<owner>/<name>/releases.atom → `latest_version` / `latest_release_at` (VERSION_RELEASED emitted by the writer on change)9 + one RELEASE event per feed entry (dedupe key = release URL).10 * https://raw.githubusercontent.com/<owner>/<name>/HEAD/README.md → archived text for later LLM summarisation (deterministic: title only).1112Entity type follows the canonical kind (aiatlas.ontology.taxonomy.normalize_framework_kind): `agent` for agents (with `agent_kind`),13`tool` for tool | application | mcp-server, `repository` for model-code releases, `framework` for everything else (libraries, SDKs,14inference engines…). Identifiers `{"github_repo": "owner/name"}` (+ `pypi` when the seed maps a package) so the PyPI connector merges15into the same entity. `kind` is the canonical value, `kind_raw` the seed label when it differed; `license` is the ontology key.16"""17from __future__ import annotations1819import re20from datetime import UTC21from typing import Any2223from aiatlas.ontology.licenses import normalize_license24from aiatlas.ontology.taxonomy import normalize_framework_kind25from aiatlas.registry import load, org_by_github, org_ref, organizations26from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext27from aiatlas.sdk.extract.dates import parse_datetime28from aiatlas.sdk.extract.feeds import FeedItem29from aiatlas.sdk.extract.html import find_in_json30from aiatlas.sdk.facts import EntityRef, Facts, Target31from aiatlas.sdk.fetch import FetchResult3233GH = "https://github.com"34RAW = "https://raw.githubusercontent.com"35TOOL_KINDS = {"tool", "application", "mcp-server"}36VERSION_TAG = re.compile(r"^v?(\d+(?:\.\d+)+(?:[-+.][0-9A-Za-z.]+)?)$")373839def repo_entries() -> list[dict[str, Any]]:40 return load("repositories")414243def entity_type_for(kind: str | None) -> str:44 """Canonical kind → entity type (agents and tools get their own surfaces; model code stays a repository)."""45 canon = normalize_framework_kind(kind) if kind else None46 if kind == "model":47 return "repository"48 if canon == "agent":49 return "agent"50 if canon in TOOL_KINDS:51 return "tool"52 return "framework"535455def claim_license(facts: Facts, ref: EntityRef, raw: str | None, *, prop: str = "license") -> str | None:56 if not raw:57 return None58 key = normalize_license(raw)59 facts.claim(ref, prop, key or raw)60 if key != raw:61 facts.claim(ref, f"{prop}_raw", raw)62 return key636465def repo_entity(facts: Facts, entry: dict[str, Any]) -> EntityRef:66 """One EntityRef per seed repository — shared by the GitHub and PyPI connectors so both resolve to the same row."""67 owner, name = entry["repo"].split("/", 1)68 kind = entry.get("kind")69 etype = entity_type_for(kind)70 for e in facts.entities:71 if e.entity_type == etype and e.identifiers.get("github_repo") == entry["repo"]:72 return e73 ids = {"github_repo": entry["repo"]}74 if entry.get("pypi"):75 ids["pypi"] = entry["pypi"]76 org = None77 if entry.get("organization") in organizations():78 org = org_ref(entry["organization"])79 else:80 known = org_by_github(owner)81 if known:82 org = org_ref(known["key"])83 aliases = [entry["repo"], entry["key"]] + ([entry["pypi"]] if entry.get("pypi") else [])84 ref = facts.entity(etype, name, identifiers=ids, organization=org, aliases=[a for a in aliases if a != name], slug_hint=entry["key"], identity_confidence="high")85 canon = normalize_framework_kind(kind) if kind and kind != "model" else kind86 facts.claim(ref, "kind", canon or kind)87 if canon and canon != kind:88 facts.claim(ref, "kind_raw", kind)89 if etype == "agent" and entry.get("agent_kind"):90 facts.claim(ref, "agent_kind", entry["agent_kind"])91 facts.claim(ref, "repository_url", f"{GH}/{entry['repo']}")92 if entry.get("pypi"):93 facts.claim(ref, "pypi", entry["pypi"])94 if org:95 facts.entities.append(org)96 facts.relate(org, "develops", ref)97 return ref9899100class GitHubConnector(BaseConnector):101 name = "github"102 label = "GitHub — AI frameworks, runtimes, agents and model repositories"103 description = "Repository pages, release feeds and raw READMEs of the seed list in registry/repositories.yaml (HTML only, no API)."104 source_key = "github.com"105 version = "2"106 parser_version = "2"107 interval_seconds = 12 * 3600108 min_interval_seconds = 6 * 3600109 max_interval_seconds = 3 * 86400110 rate_per_min = 20111 tier = 2112 priority = 1113 expected_min_records = 40114 concurrency = 2115116 async def discover(self, ctx: RunContext) -> list[Target]:117 out: list[Target] = []118 for entry in repo_entries():119 repo = entry["repo"]120 meta = {"repo": repo, "key": entry["key"]}121 out.append(Target(url=f"{GH}/{repo}", doc_type="repository", key=f"repo:{repo}", meta=meta, min_bytes=20000, priority=1))122 out.append(Target(url=f"{GH}/{repo}/releases.atom", doc_type="feed", key=f"releases:{repo}", meta=meta, min_bytes=200, priority=2))123 out.append(Target(url=f"{RAW}/{repo}/HEAD/README.md", doc_type="readme", key=f"readme:{repo}", meta={**meta, "content_type": "text/markdown",124 "llm_task": "repository_summary"}, min_bytes=200, priority=3, rate_per_min=30))125 return out126127 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:128 facts = Facts()129 entry = next((e for e in repo_entries() if e["repo"] == target.meta.get("repo")), None)130 if not entry:131 return facts132 ref = repo_entity(facts, entry)133 key = target.key or ""134 if key.startswith("repo:") and parsed.html:135 self._repo_page(facts, ref, entry, parsed)136 elif key.startswith("releases:") and parsed.kind == "feed":137 self._releases(facts, ref, entry, parsed.feed_items)138 elif key.startswith("readme:") and parsed.markdown:139 h1 = next((t for lvl, t in parsed.markdown.headings if lvl == 1), None)140 facts.claim(ref, "readme_url", target.url)141 facts.document_title = h1 or f"{entry['repo']} README"142 facts.document_entity = ref143 return facts144145 # ------------------------------------------------------------------------------------------ repository page146 def _repo_page(self, facts: Facts, ref: EntityRef, entry: dict[str, Any], parsed: Parsed) -> None:147 html = parsed.html148 assert html149 about: dict[str, Any] = {}150 repo_meta: dict[str, Any] = {}151 for blob in html.embedded_json.values():152 hits = find_in_json(blob, "sidebarAbout", max_hits=1)153 if hits and isinstance(hits[0], dict):154 about = hits[0]155 layout = find_in_json(blob, "codeViewLayoutRoute", max_hits=1)156 if layout and isinstance(layout[0], dict) and isinstance(layout[0].get("repo"), dict):157 repo_meta = layout[0]["repo"]158 break159 description = about.get("description") or _og_description(html, entry["repo"])160 facts.claim(ref, "description", description[:2000] if description else None)161 website = about.get("website")162 facts.claim(ref, "homepage", website if isinstance(website, str) and website.startswith("http") else None)163 topics = [t.get("name") for t in about.get("topics") or [] if isinstance(t, dict) and t.get("name")]164 if not topics:165 topics = [href.rsplit("/topics/", 1)[1] for href, _ in html.links if "/topics/" in href][:30]166 facts.claim(ref, "topics", topics[:30])167 stars = about.get("stargazerCount")168 if stars is None:169 stars = _counter(html, "#repo-stars-counter-star")170 forks = about.get("forksCount")171 if forks is None:172 forks = _counter(html, "#repo-network-counter")173 facts.claim(ref, "metric.stars", stars)174 facts.claim(ref, "metric.forks", forks)175 facts.claim(ref, "metric.watchers", about.get("watcherCount"))176 sections = about.get("sections") or {}177 rel = sections.get("releases") if isinstance(sections, dict) else None178 if isinstance(rel, dict):179 facts.claim(ref, "metric.releases", rel.get("releaseCount"))180 facts.claim(ref, "metric.tags", rel.get("tagCount"))181 lic = ((about.get("repo") or {}).get("license") or {}) if isinstance(about.get("repo"), dict) else {}182 spdx = lic.get("spdxId") if isinstance(lic, dict) else None183 if not spdx:184 node = next((n for n in html.css("[data-content]") if (n.attributes.get("data-content") or "").endswith(" license")), None)185 spdx = node.attributes["data-content"].removesuffix(" license").strip() if node else None186 claim_license(facts, ref, spdx if spdx and spdx.upper() != "NOASSERTION" else None)187 facts.claim(ref, "license_name", lic.get("name") if isinstance(lic, dict) else None)188 created = parse_datetime(repo_meta.get("createdAt")) if repo_meta.get("createdAt") else None189 facts.claim(ref, "created_at", created.astimezone(UTC).isoformat(timespec="seconds") if created else None)190 facts.claim(ref, "default_branch", repo_meta.get("defaultBranch"))191 arch = (about.get("repo") or {}).get("isArchived") if isinstance(about.get("repo"), dict) else None192 if arch is True:193 facts.claim(ref, "status", "archived")194 facts.document_title = html.title195196197 # ------------------------------------------------------------------------------------------ releases feed198 def _releases(self, facts: Facts, ref: EntityRef, entry: dict[str, Any], items: list[FeedItem]) -> None:199 items = [i for i in items if i.url and i.title]200 if not items:201 return202 items.sort(key=lambda i: (i.updated_at or i.published_at or _EPOCH), reverse=True)203 latest = next((i for i in items if VERSION_TAG.match(i.title.strip())), items[0])204 m = VERSION_TAG.match(latest.title.strip())205 facts.claim(ref, "latest_version", m.group(1) if m else latest.title.strip()[:100])206 facts.claim(ref, "latest_release_tag", latest.title.strip()[:100])207 when = latest.updated_at or latest.published_at208 facts.claim(ref, "latest_release_at", when.astimezone(UTC).isoformat(timespec="seconds") if when else None)209 facts.claim(ref, "releases_url", f"{GH}/{entry['repo']}/releases")210 for it in items[:30]:211 when = it.updated_at or it.published_at212 facts.event("RELEASE", "release", f"{entry['repo']} released {it.title.strip()}", entity=ref, importance=1, effective_at=when,213 dedupe_key=f"RELEASE:{it.url}", source_url=it.url,214 meta={"tag": it.title.strip()[:100], "authors": it.authors[:3], "summary": (it.summary or "")[:300]})215 facts.document_title = f"{entry['repo']} releases"216217218def _counter(html: Any, selector: str) -> int | None:219 node = html.css_first(selector)220 if node is None:221 return None222 raw = (node.attributes.get("title") or node.text(strip=True) or "").replace(",", "")223 return int(raw) if raw.isdigit() else None224225226def _og_description(html: Any, repo: str) -> str | None:227 d = html.og.get("og:description") or html.description or ""228 d = re.sub(rf"\s*-\s*{re.escape(repo)}\s*$", "", d).strip()229 return d or None230231232from datetime import datetime233234_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)235236CONNECTORS = [GitHubConnector]237