1"""GitHub — seed list of AI repositories (registry/repositories.yaml), server-rendered pages only (no API).23Per repository (tier 2):4 * https://github.com/<owner>/<name> → the repository overview embeds `react-app.embeddedData` JSON (`payload.sidebarAbout`:5 description, website, topics, stargazerCount, forksCount, license, release count; `repo.createdAt`).6 CSS fallbacks: `#repo-stars-counter-star[title]`, `#repo-network-counter[title]`,7 `[data-content$=" license"]`, `a[href^="/topics/"]`. The language bar is client-loaded → no `language` claim.8 * https://github.com/<owner>/<name>/releases.atom → `latest_version` / `latest_release_at` (VERSION_RELEASED emitted by the writer on change)9 + one RELEASE event per feed entry (dedupe key = release URL).10 * https://raw.githubusercontent.com/<owner>/<name>/HEAD/README.md → archived text for later LLM summarisation (deterministic: title only).1112Entity type: `framework` for kind framework|runtime|library|agent|tool, `repository` otherwise (model code drops). Identifiers13`{"github_repo": "owner/name"}` (+ `pypi` when the seed maps a package) so the PyPI connector merges into the same entity.14"""15from __future__ import annotations1617import re18from datetime import UTC19from typing import Any2021from aiatlas.registry import load, org_by_github, org_ref, organizations22from aiatlas.sdk.connector import BaseConnector, Parsed, RunContext23from aiatlas.sdk.extract.dates import parse_datetime24from aiatlas.sdk.extract.feeds import FeedItem25from aiatlas.sdk.extract.html import find_in_json26from aiatlas.sdk.facts import EntityRef, Facts, Target27from aiatlas.sdk.fetch import FetchResult2829GH = "https://github.com"30RAW = "https://raw.githubusercontent.com"31FRAMEWORK_KINDS = {"framework", "runtime", "library", "agent", "tool"}32VERSION_TAG = re.compile(r"^v?(\d+(?:\.\d+)+(?:[-+.][0-9A-Za-z.]+)?)$")333435def repo_entries() -> list[dict[str, Any]]:36 return load("repositories")373839def repo_entity(facts: Facts, entry: dict[str, Any]) -> EntityRef:40 """One EntityRef per seed repository — shared by the GitHub and PyPI connectors so both resolve to the same row."""41 owner, name = entry["repo"].split("/", 1)42 etype = "framework" if entry.get("kind") in FRAMEWORK_KINDS else "repository"43 ids = {"github_repo": entry["repo"]}44 if entry.get("pypi"):45 ids["pypi"] = entry["pypi"]46 org = None47 if entry.get("organization") in organizations():48 org = org_ref(entry["organization"])49 else:50 known = org_by_github(owner)51 if known:52 org = org_ref(known["key"])53 aliases = [entry["repo"], entry["key"]] + ([entry["pypi"]] if entry.get("pypi") else [])54 ref = facts.entity(etype, name, identifiers=ids, organization=org, aliases=[a for a in aliases if a != name], slug_hint=entry["key"])55 facts.claim(ref, "kind", entry.get("kind"))56 facts.claim(ref, "repository_url", f"{GH}/{entry['repo']}")57 if entry.get("pypi"):58 facts.claim(ref, "pypi", entry["pypi"])59 if org:60 facts.entities.append(org)61 facts.relate(org, "develops", ref)62 return ref636465class GitHubConnector(BaseConnector):66 name = "github"67 label = "GitHub — AI frameworks, runtimes, agents and model repositories"68 description = "Repository pages, release feeds and raw READMEs of the seed list in registry/repositories.yaml (HTML only, no API)."69 source_key = "github.com"70 version = "1"71 parser_version = "1"72 interval_seconds = 12 * 360073 min_interval_seconds = 6 * 360074 max_interval_seconds = 3 * 8640075 rate_per_min = 2076 tier = 277 priority = 178 expected_min_records = 4079 concurrency = 28081 async def discover(self, ctx: RunContext) -> list[Target]:82 out: list[Target] = []83 for entry in repo_entries():84 repo = entry["repo"]85 meta = {"repo": repo, "key": entry["key"]}86 out.append(Target(url=f"{GH}/{repo}", doc_type="repository", key=f"repo:{repo}", meta=meta, min_bytes=20000, priority=1))87 out.append(Target(url=f"{GH}/{repo}/releases.atom", doc_type="feed", key=f"releases:{repo}", meta=meta, min_bytes=200, priority=2))88 out.append(Target(url=f"{RAW}/{repo}/HEAD/README.md", doc_type="readme", key=f"readme:{repo}", meta={**meta, "content_type": "text/markdown",89 "llm_task": "repository_summary"}, min_bytes=200, priority=3, rate_per_min=30))90 return out9192 async def extract(self, ctx: RunContext, target: Target, res: FetchResult, parsed: Parsed) -> Facts:93 facts = Facts()94 entry = next((e for e in repo_entries() if e["repo"] == target.meta.get("repo")), None)95 if not entry:96 return facts97 ref = repo_entity(facts, entry)98 key = target.key or ""99 if key.startswith("repo:") and parsed.html:100 self._repo_page(facts, ref, entry, parsed)101 elif key.startswith("releases:") and parsed.kind == "feed":102 self._releases(facts, ref, entry, parsed.feed_items)103 elif key.startswith("readme:") and parsed.markdown:104 h1 = next((t for lvl, t in parsed.markdown.headings if lvl == 1), None)105 facts.claim(ref, "readme_url", target.url)106 facts.document_title = h1 or f"{entry['repo']} README"107 facts.document_entity = ref108 return facts109110 # ------------------------------------------------------------------------------------------ repository page111 def _repo_page(self, facts: Facts, ref: EntityRef, entry: dict[str, Any], parsed: Parsed) -> None:112 html = parsed.html113 assert html114 about: dict[str, Any] = {}115 repo_meta: dict[str, Any] = {}116 for blob in html.embedded_json.values():117 hits = find_in_json(blob, "sidebarAbout", max_hits=1)118 if hits and isinstance(hits[0], dict):119 about = hits[0]120 layout = find_in_json(blob, "codeViewLayoutRoute", max_hits=1)121 if layout and isinstance(layout[0], dict) and isinstance(layout[0].get("repo"), dict):122 repo_meta = layout[0]["repo"]123 break124 description = about.get("description") or _og_description(html, entry["repo"])125 facts.claim(ref, "description", description[:2000] if description else None)126 website = about.get("website")127 facts.claim(ref, "homepage", website if isinstance(website, str) and website.startswith("http") else None)128 topics = [t.get("name") for t in about.get("topics") or [] if isinstance(t, dict) and t.get("name")]129 if not topics:130 topics = [href.rsplit("/topics/", 1)[1] for href, _ in html.links if "/topics/" in href][:30]131 facts.claim(ref, "topics", topics[:30])132 stars = about.get("stargazerCount")133 if stars is None:134 stars = _counter(html, "#repo-stars-counter-star")135 forks = about.get("forksCount")136 if forks is None:137 forks = _counter(html, "#repo-network-counter")138 facts.claim(ref, "metric.stars", stars)139 facts.claim(ref, "metric.forks", forks)140 facts.claim(ref, "metric.watchers", about.get("watcherCount"))141 sections = about.get("sections") or {}142 rel = sections.get("releases") if isinstance(sections, dict) else None143 if isinstance(rel, dict):144 facts.claim(ref, "metric.releases", rel.get("releaseCount"))145 facts.claim(ref, "metric.tags", rel.get("tagCount"))146 lic = ((about.get("repo") or {}).get("license") or {}) if isinstance(about.get("repo"), dict) else {}147 spdx = lic.get("spdxId") if isinstance(lic, dict) else None148 if not spdx:149 node = next((n for n in html.css("[data-content]") if (n.attributes.get("data-content") or "").endswith(" license")), None)150 spdx = node.attributes["data-content"].removesuffix(" license").strip() if node else None151 facts.claim(ref, "license", spdx if spdx and spdx.upper() != "NOASSERTION" else None)152 facts.claim(ref, "license_name", lic.get("name") if isinstance(lic, dict) else None)153 created = parse_datetime(repo_meta.get("createdAt")) if repo_meta.get("createdAt") else None154 facts.claim(ref, "created_at", created.astimezone(UTC).isoformat(timespec="seconds") if created else None)155 facts.claim(ref, "default_branch", repo_meta.get("defaultBranch"))156 arch = (about.get("repo") or {}).get("isArchived") if isinstance(about.get("repo"), dict) else None157 if arch is True:158 facts.claim(ref, "status", "archived")159 facts.document_title = html.title160161162 # ------------------------------------------------------------------------------------------ releases feed163 def _releases(self, facts: Facts, ref: EntityRef, entry: dict[str, Any], items: list[FeedItem]) -> None:164 items = [i for i in items if i.url and i.title]165 if not items:166 return167 items.sort(key=lambda i: (i.updated_at or i.published_at or _EPOCH), reverse=True)168 latest = next((i for i in items if VERSION_TAG.match(i.title.strip())), items[0])169 m = VERSION_TAG.match(latest.title.strip())170 facts.claim(ref, "latest_version", m.group(1) if m else latest.title.strip()[:100])171 facts.claim(ref, "latest_release_tag", latest.title.strip()[:100])172 when = latest.updated_at or latest.published_at173 facts.claim(ref, "latest_release_at", when.astimezone(UTC).isoformat(timespec="seconds") if when else None)174 facts.claim(ref, "releases_url", f"{GH}/{entry['repo']}/releases")175 for it in items[:30]:176 when = it.updated_at or it.published_at177 facts.event("RELEASE", "release", f"{entry['repo']} released {it.title.strip()}", entity=ref, importance=1, effective_at=when,178 dedupe_key=f"RELEASE:{it.url}", source_url=it.url,179 meta={"tag": it.title.strip()[:100], "authors": it.authors[:3], "summary": (it.summary or "")[:300]})180 facts.document_title = f"{entry['repo']} releases"181182183def _counter(html: Any, selector: str) -> int | None:184 node = html.css_first(selector)185 if node is None:186 return None187 raw = (node.attributes.get("title") or node.text(strip=True) or "").replace(",", "")188 return int(raw) if raw.isdigit() else None189190191def _og_description(html: Any, repo: str) -> str | None:192 d = html.og.get("og:description") or html.description or ""193 d = re.sub(rf"\s*-\s*{re.escape(repo)}\s*$", "", d).strip()194 return d or None195196197from datetime import datetime198199_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)200201CONNECTORS = [GitHubConnector]202