TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Generate config/sources.d/57-packages-top.yaml — `package` connector sensors for the most downloaded packages3of the main software registries, following the conventions of 31-packages.yaml (one sensor per (registry,4package); registries that already exist as sources are `extend: true`d; new registry sources are declared with5`llm: false`; tier D everywhere — version streams are a firehose, heuristics only).67Lists (public, reproducible — the retrieval date is written in the fragment header):8 PyPI https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json (monthly top 15 000 by downloads)9 npm https://github.com/tristan-f-r/npm-rank/releases/download/latest/raw.json (top 10 000 by downloads,10 ordered by popularity; successor of anvaka/npmrank — sample/topDependents.md is gone)11 crates.io https://crates.io/api/v1/crates?sort=downloads&per_page=100&page=1..3 (User-Agent required)12 RubyGems https://bestgems.org/total?page=1..5 (total downloads ranking, 20 gems per page; rubygems.org's13 /api/v1/downloads/all.json only returns the 50 most downloaded *versions*)14 NuGet https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc15 Docker Hub https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count (official images;16 `-pull_count` sorts ascending on the current API and page 2 needs a login)17 Go GitHub search `language:go stars:>5000` sorted by stars (pages 1–3) → module path read from the18 repository's go.mod (index.golang.org is a chronological feed, not a ranking); repositories whose19 go.mod has no domain-qualified module path are dropped.2021Usage: python3 scripts/gen-packages-top.py [--cache /tmp/pkgtop] [--out config/sources.d/57-packages-top.yaml]22Packages already monitored anywhere in config/ (any `connector: package` sensor) are skipped. Afterwards run the23registry validator and prune everything that is not OK:24 node node_modules/tsx/dist/cli.mjs apps/engine/src/validate.ts config/sources.d/57-packages-top.yaml --concurrency 8 --json /tmp/pkg-report.json --quiet25 python3 scripts/prune-fragment.py config/sources.d/57-packages-top.yaml /tmp/pkg-report.json --drop-warn --keep-empty26"""27from __future__ import annotations2829import argparse30import concurrent.futures as cf31import datetime as dt32import json33import os34import pathlib35import re36import subprocess37import sys38import urllib.request3940ROOT = pathlib.Path(__file__).resolve().parents[1]41UA = "WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)"42LIMITS = {"pypi": 1200, "npm": 1200, "crates": 300, "rubygems": 100, "nuget": 100, "dockerhub": 100, "goproxy": 150}4344RESERVED = {"y", "n", "yes", "no", "on", "off", "true", "false", "null", "~"}454647def fetch(url: str, cache: pathlib.Path, name: str, headers: dict[str, str] | None = None) -> bytes:48 p = cache / name49 if p.exists() and p.stat().st_size > 0:50 return p.read_bytes()51 req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json, text/html;q=0.8, */*;q=0.5", **(headers or {})})52 with urllib.request.urlopen(req, timeout=60) as r:53 data = r.read()54 p.write_bytes(data)55 return data565758def yq(s: str, allow_space: bool = False) -> str:59 """Quote a YAML flow scalar when it is not a safe plain scalar (`?`, `@…`, numbers, `...`, booleans…)."""60 pat = r"[A-Za-z][A-Za-z0-9 _.@+/-]*" if allow_space else r"[A-Za-z_][A-Za-z0-9_.+/-]*"61 if re.fullmatch(pat, s) and s.lower() not in RESERVED and not re.fullmatch(r"[0-9.eE+-]+", s) and not s.endswith(".") and "..." not in s:62 return s63 return json.dumps(s)646566def slug(s: str) -> str:67 return re.sub(r"^-+|-+$", "", re.sub(r"[^a-z0-9]+", "-", s.lower()))[:80]686970def existing_packages() -> set[tuple[str, str]]:71 seen: set[tuple[str, str]] = set()72 rx = re.compile(r"connector:\s*package.*?registry:\s*([a-z]+),\s*name:\s*\"?([^,\"}\s]+)")73 for f in [ROOT / "config" / "sources.yaml", *sorted((ROOT / "config" / "sources.d").glob("*.yaml"))]:74 if f.name == "57-packages-top.yaml":75 continue76 for m in rx.finditer(f.read_text()):77 seen.add((m.group(1), m.group(2).lower()))78 return seen798081# ── list loaders ─────────────────────────────────────────────────────────────────────────────────────────────8283def list_pypi(cache: pathlib.Path) -> list[str]:84 d = json.loads(fetch("https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json", cache, "pypi.json"))85 return [r["project"] for r in d["rows"]]868788def list_npm(cache: pathlib.Path) -> list[str]:89 d = json.loads(fetch("https://github.com/tristan-f-r/npm-rank/releases/download/latest/raw.json", cache, "npm-rank.json"))90 return [r["name"] for r in d]919293def list_crates(cache: pathlib.Path) -> list[str]:94 out: list[str] = []95 for p in (1, 2, 3):96 d = json.loads(fetch(f"https://crates.io/api/v1/crates?sort=downloads&per_page=100&page={p}", cache, f"crates-{p}.json"))97 out += [c["id"] for c in d["crates"]]98 return out99100101def list_rubygems(cache: pathlib.Path) -> list[str]:102 out: list[str] = []103 for p in range(1, 6):104 h = fetch(f"https://bestgems.org/total?page={p}", cache, f"bestgems-{p}.html").decode("utf8", "replace")105 for m in re.finditer(r'href="/gems/([A-Za-z0-9_.-]+)"', h):106 if m.group(1) not in out:107 out.append(m.group(1))108 return out109110111def list_nuget(cache: pathlib.Path) -> list[str]:112 d = json.loads(fetch("https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc", cache, "nuget.json"))113 return [x["id"] for x in d["data"]]114115116def list_dockerhub(cache: pathlib.Path) -> list[str]:117 d = json.loads(fetch("https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count", cache, "dockerhub.json"))118 rows = sorted(d["results"], key=lambda x: -(x.get("pull_count") or 0))119 return [x["name"] for x in rows]120121122def list_go(cache: pathlib.Path) -> list[str]:123 items: list[dict] = []124 for p in (1, 2, 3):125 d = json.loads(fetch(f"https://api.github.com/search/repositories?q=language:go+stars:>5000&sort=stars&order=desc&per_page=100&page={p}", cache, f"gh-go-{p}.json", {"Accept": "application/vnd.github+json"}))126 items += d.get("items", [])127 modfile = cache / "go-modules.json"128 if modfile.exists():129 mods = {r["repo"]: r["module"] for r in json.loads(modfile.read_text())}130 else:131 def one(it: dict) -> tuple[str, str | None]:132 fn, br = it["full_name"], it.get("default_branch", "HEAD")133 try:134 txt = subprocess.run(["curl", "-sL", "--max-time", "20", f"https://raw.githubusercontent.com/{fn}/{br}/go.mod"], capture_output=True, text=True, timeout=30).stdout135 except Exception:136 return fn, None137 m = re.search(r"^module\s+(\S+)", txt, re.M)138 return fn, m.group(1) if m else None139 with cf.ThreadPoolExecutor(16) as ex:140 res = list(ex.map(one, items))141 mods = dict(res)142 modfile.write_text(json.dumps([{"repo": r, "module": m, "stars": i["stargazers_count"]} for (r, m), i in zip(res, items)]))143 out: list[str] = []144 for it in items:145 m = mods.get(it["full_name"])146 if not m or "." not in m.split("/")[0] or len(m.split("/")) < 2:147 continue # `module ragflow`, `gitea.dev`… not a fetchable module path148 if m not in out:149 out.append(m)150 return out151152153# ── emitters ─────────────────────────────────────────────────────────────────────────────────────────────────154155def sensor(registry: str, name: str) -> str:156 label = {"pypi": "pypi", "npm": "npm", "crates": "crates", "rubygems": "rubygems", "nuget": "nuget", "dockerhub": "docker hub", "goproxy": "go"}[registry]157 page = {158 "pypi": f"https://pypi.org/project/{name}/",159 "npm": f"https://www.npmjs.com/package/{name}",160 "crates": f"https://crates.io/crates/{name}",161 "rubygems": f"https://rubygems.org/gems/{name}",162 "nuget": f"https://www.nuget.org/packages/{name}",163 "dockerhub": f"https://hub.docker.com/_/{name}",164 "goproxy": f"https://pkg.go.dev/{name}",165 }[registry]166 cfg_name = f"library/{name}" if registry == "dockerhub" else name167 return f' - {{ name: {yq(f"{label} {name}", True)}, url: "{page}", type: REST_API, connector: package, tier: D, config: {{ registry: {registry}, name: {yq(cfg_name)}, maxItems: 30 }} }}'168169170def main() -> None:171 ap = argparse.ArgumentParser()172 ap.add_argument("--cache", default="/tmp/pkgtop")173 ap.add_argument("--out", default=str(ROOT / "config" / "sources.d" / "57-packages-top.yaml"))174 a = ap.parse_args()175 cache = pathlib.Path(a.cache)176 cache.mkdir(parents=True, exist_ok=True)177 skip = existing_packages()178 today = dt.date.today().isoformat()179180 lists = {"pypi": list_pypi(cache), "npm": list_npm(cache), "crates": list_crates(cache), "rubygems": list_rubygems(cache), "nuget": list_nuget(cache), "dockerhub": list_dockerhub(cache), "goproxy": list_go(cache)}181 chosen: dict[str, list[str]] = {}182 skipped: dict[str, int] = {}183 for reg, names in lists.items():184 out: list[str] = []185 slugs: set[str] = set()186 skipped[reg] = 0187 for n in names:188 key = (reg, (f"library/{n}" if reg == "dockerhub" else n).lower())189 if key in skip:190 skipped[reg] += 1191 continue192 s = slug(f"{reg} {n}")193 if s in slugs:194 continue # `foo-bar` vs `foo_bar` would collide on the derived sensor id195 slugs.add(s)196 out.append(n)197 if len(out) >= LIMITS[reg]:198 break199 chosen[reg] = out200201 L: list[str] = []202 L.append(f"# config/sources.d/57-packages-top.yaml — the most downloaded packages of the main software registries")203 L.append(f"# ({today}, `package` connector, tier D, heuristics only): the version streams of the software supply chain")204 L.append("# beyond the ~100 packages attached to their project entities in 31-packages.yaml. Sensors are attached to the")205 L.append("# registry's own organization (npm Registry, PyPI, Docker Hub → docker, Go module proxy → go are `extend: true`d;")206 L.append("# crates.io, RubyGems.org and NuGet Gallery are declared here with `llm: false` — the extended sources keep the")207 L.append("# `llm` of their declaring file). Every sensor was fetched and parsed by `apps/engine/src/validate.ts` before")208 L.append("# being written (packages returning WARN/FAIL were pruned). Generated by `scripts/gen-packages-top.py`; lists:")209 L.append(f"# PyPI top {LIMITS['pypi']} by downloads — https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json")210 L.append(f"# npm top {LIMITS['npm']} by downloads — https://github.com/tristan-f-r/npm-rank (releases/download/latest/raw.json)")211 L.append(f"# crates.io top {LIMITS['crates']} — https://crates.io/api/v1/crates?sort=downloads&per_page=100&page=1..3")212 L.append(f"# RubyGems top {LIMITS['rubygems']} total downloads — https://bestgems.org/total?page=1..5")213 L.append(f"# NuGet top {LIMITS['nuget']} — https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc")214 L.append(f"# Docker Hub top {LIMITS['dockerhub']} official images — https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count")215 L.append(f"# Go top {LIMITS['goproxy']} modules — GitHub search language:go stars:>5000 sorted by stars, module path from go.mod")216 L.append("# Packages already monitored elsewhere in config/ were skipped: " + ", ".join(f"{k} {v}" for k, v in skipped.items() if v) + ".")217 L.append("sources:")218219 L.append(" # ── Python (PyPI) ──")220 L.append(" - id: pypi")221 L.append(" extend: true")222 L.append(" categories: [packages]")223 L.append(" sensors:")224 L += [sensor("pypi", n) for n in chosen["pypi"]]225226 L.append(" # ── JavaScript / TypeScript (npm) ──")227 L.append(" - id: npm")228 L.append(" extend: true")229 L.append(" categories: [packages]")230 L.append(" sensors:")231 L += [sensor("npm", n) for n in chosen["npm"]]232233 L.append(" # ── Rust (crates.io) ──")234 L.append(" - id: crates-io")235 L.append(" name: crates.io")236 L.append(" domain: crates.io")237 L.append(" homepage: https://crates.io")238 L.append(" categories: [packages, developer, infrastructure, open-source]")239 L.append(" tier: D")240 L.append(" weight: 1.2")241 L.append(" country: US")242 L.append(" language: en")243 L.append(" aliases: [crates.io, crates, cargo registry, rust package registry]")244 L.append(" discover: { rss: true, status: true }")245 L.append(" llm: false")246 L.append(' notes: "The Rust Foundation operates crates.io; the API asks for a User-Agent and ~1 request/s (validate at --concurrency 1)."')247 L.append(" sensors:")248 L += [sensor("crates", n) for n in chosen["crates"]]249250 L.append(" # ── Ruby (RubyGems) ──")251 L.append(" - id: rubygems")252 L.append(" name: RubyGems.org")253 L.append(" domain: rubygems.org")254 L.append(" homepage: https://rubygems.org")255 L.append(" categories: [packages, developer, infrastructure, open-source]")256 L.append(" tier: D")257 L.append(" weight: 1.1")258 L.append(" country: US")259 L.append(" language: en")260 L.append(" aliases: [rubygems, rubygems.org, ruby central, gem registry]")261 L.append(" discover: { rss: true, status: true }")262 L.append(" llm: false")263 L.append(" sensors:")264 L += [sensor("rubygems", n) for n in chosen["rubygems"]]265266 L.append(" # ── .NET (NuGet) ──")267 L.append(" - id: nuget")268 L.append(" name: NuGet Gallery")269 L.append(" domain: nuget.org")270 L.append(" homepage: https://www.nuget.org")271 L.append(" categories: [packages, developer, infrastructure, open-source]")272 L.append(" tier: D")273 L.append(" weight: 1.1")274 L.append(" country: US")275 L.append(" language: en")276 L.append(" aliases: [nuget, nuget.org, nuget gallery, .net package registry]")277 L.append(" discover: { rss: true, status: true }")278 L.append(" llm: false")279 L.append(" sensors:")280 L += [sensor("nuget", n) for n in chosen["nuget"]]281282 L.append(" # ── Containers (Docker Hub official images) ──")283 L.append(" - id: docker")284 L.append(" extend: true")285 L.append(" categories: [packages]")286 L.append(" sensors:")287 L += [sensor("dockerhub", n) for n in chosen["dockerhub"]]288289 L.append(" # ── Go (module proxy) ──")290 L.append(" - id: go")291 L.append(" extend: true")292 L.append(" categories: [packages]")293 L.append(" sensors:")294 L += [sensor("goproxy", n) for n in chosen["goproxy"]]295296 pathlib.Path(a.out).write_text("\n".join(L) + "\n")297 total = sum(len(v) for v in chosen.values())298 print(f"wrote {a.out}: {total} sensors — " + ", ".join(f"{k} {len(v)} (skipped {skipped[k]} existing, list {len(lists[k])})" for k, v in chosen.items()), file=sys.stderr)299300301if __name__ == "__main__":302 main()303