TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Generate config/sources.d/47-open-source-long-tail.yaml from the compact spec in3scripts/gen-open-source-long-tail-spec*.py (six parts, executed in order).45Usage:6 python3 scripts/gen-open-source-long-tail.py # write the fragment7 python3 scripts/gen-open-source-long-tail.py --exclude /tmp/v47.json8 # re-generate without every URL the validator reported as FAIL or WARN (remembered in /tmp/v47-bad.json)9 python3 scripts/gen-open-source-long-tail.py --flip /tmp/v47.json10 # re-generate, turning every `releases.atom` that the validator reported as11 # `WARN empty list` into a `tags.atom` sensor (kind: tags); flips are remembered in12 # /tmp/v47-flips.json so several passes accumulate.1314Safety: repos / URLs already present anywhere else in config/ are skipped, and a new (non-extend)15id that already exists in another file aborts the run.16"""17import json18import os19import re20import sys2122ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))23OUT = os.path.join(ROOT, "config/sources.d/47-open-source-long-tail.yaml")24FLIPS = "/tmp/v47-flips.json"25BAD = "/tmp/v47-bad.json"26# Keep entries reviewable and the fragment within its 350–500 sensor budget: at most this many GitHub repos per27# organization unless listed in REPO_CAPS; only ids in KEEP_IDS enter the fragment (the rest of the spec is kept for a later fragment).28DEFAULT_REPO_CAP = 129EXTEND_REPO_CAP = 130REPO_CAPS = {'apache': 26, 'cncf': 30, 'python': 4, 'nodejs': 3, 'go': 3, 'rust': 3, 'java-libraries': 6, 'react': 3, 'dotnet': 3, 'kotlin': 2, 'swift': 2, 'ruby': 3, 'php': 3, 'google': 4, 'aws': 3, 'red-hat': 3, 'meta-open-source': 3, 'postgresql': 3, 'openjs': 6, 'eclipse': 3, 'microsoft': 2, 'rust-foundation': 2, 'scala': 2, 'elixir': 2, 'vue': 2, 'angular': 2, 'tailwind': 2, 'spring': 2, 'laravel': 2, 'webassembly': 3, 'openinfra': 2, 'openssf': 2, 'docker': 2, 'hashicorp': 2, 'ubuntu': 2, 'github': 2, 'gnome': 2, 'kde': 2, 'osgeo': 3, 'openapi-tools': 3, 'terraform-tooling': 2, 'freedesktop': 1}31KEEP_IDS = set("""32linux-foundation numfocus fsfe openinfra bytecode-alliance webassembly openjs eclipse apache cncf openssf33rust-foundation rust python astral nodejs go ruby rails php symfony laravel spring quarkus micronaut openjdk graalvm java-libraries kotlin swift elixir erlang gleam scala haskell ocaml dart flutter lua nim crystal vlang odin-lang mojo perl clojure racket julia r-project posit dotnet deno microsoft typescript34expressjs fastify nestjs hono remix nuxt svelte solidjs qwik htmx tanstack vue react react-native angular tailwind shadcn-ui radix-ui mui chakra-ui ant-design storybook nx biome oxc swc rspack vite webpack eslint prettier graphql apollo-graphql hasura prisma keycloak ory authentik zitadel cypress testing-library d3 chartjs plotly threejs maplibre osgeo mermaid35postgresql sqlite mongodb redis duckdb clickhouse elastic neo4j cassandra kafka rabbitmq airflow dbt redpanda kong caddy nginx questdb arangodb scylladb opensearch milvus pingcap yugabyte dragonflydb valkey influxdata trino debezium metabase prefect dagster appwrite quickwit emqx zeromq camunda openresty varnish swagger openapi-tools36grafana sentry datadog posthog victoriametrics netdata uptime-kuma signoz zipkin plausible matomo zabbix nagios icinga37podman opencontainers docker firecracker lima libvirt sidero-labs k0s rancher linuxcontainers ubuntu hashicorp ansible pulumi saltproject terragrunt atlantis sops github gitlab dagger flatpak renovate wireguard openvpn38gnu obsidian alacritty wezterm kitty ghostty tmux fish-shell zsh starship helix git gitea forgejo guix nixos mozilla homebrew neovim vim zed jetbrains gimp gnome kde audacity keepassxc nextcloud syncthing borgbackup jellyfin immich home-assistant openhab openwrt netgate opnsense pi-hole adguard-dns telegram jitsi mattermost matrix-ecosystem onlyoffice wikimedia zotero anki ffmpeg imagemagick freecad arduino espressif micropython raspberry-pi zephyr freertos khronos winehq libretro39wordpress woocommerce payload-cms directus keystatic medusa saleor magento prestashop shopware typo3 hugo latex-project discourse mastodon bluesky-atproto ionic expo40scikit-learn xgboost pandas polars numpy scipy matplotlib ray dask onnx lightning-ai explosion opencv intel nvidia huggingface pytorch tensorflow jax41linux-kernel systemd freedesktop hyprland sway xfce linux-mint rocky-linux almalinux void-linux freebsd openbsd netbsd haiku reactos ladybird42google meta-open-source shopify stripe alibaba-cloud red-hat suse aws astro pandoc typst43""".split())444546def load_spec():47 """Execute scripts/gen-open-source-long-tail-spec*.py in order in one namespace and concatenate SPEC, SPEC2 … SPEC6."""48 here = os.path.dirname(os.path.abspath(__file__))49 ns: dict = {}50 spec: list = []51 for n in ["", "2", "3", "4", "5", "6"]:52 exec(compile(open(os.path.join(here, f"gen-open-source-long-tail-spec{n}.py"), encoding="utf-8").read(), f"spec{n}", "exec"), ns)53 spec += ns[f"SPEC{n}"]54 return spec, ns["HEADER"]555657SPEC, HEADER = load_spec()585960def q(v: str) -> str:61 """Quote a scalar for compact flow-style YAML when it contains characters that cannot start or sit in a plain scalar."""62 if re.search(r'[:#{}\[\],&*!|>\'"%@`]|^[-?]|^\s|\s$', v) or v.lower() in {"yes", "no", "true", "false", "null", "on", "off"}:63 return json.dumps(v, ensure_ascii=False)64 return v656667def registry_text_without_fragment() -> str:68 parts = []69 cfg = os.path.join(ROOT, "config")70 for dp, _, fns in os.walk(cfg):71 for fn in fns:72 p = os.path.join(dp, fn)73 if fn.endswith(".yaml") and os.path.abspath(p) != os.path.abspath(OUT):74 parts.append(open(p, encoding="utf-8").read())75 return "\n".join(parts)767778def main() -> None:79 flips = set(json.load(open(FLIPS))) if os.path.exists(FLIPS) else set()80 if "--flip" in sys.argv:81 rep = json.load(open(sys.argv[sys.argv.index("--flip") + 1]))82 for r in rep["results"]:83 if r["status"] == "WARN" and r["note"].startswith("empty list") and r["url"].endswith("/releases.atom"):84 m = re.search(r"github\.com/([^/]+/[^/]+)/releases\.atom", r["url"])85 if m:86 flips.add(m.group(1).lower())87 json.dump(sorted(flips), open(FLIPS, "w"))88 print(f"{len(flips)} repos flipped to tags", file=sys.stderr)8990 bad = set(json.load(open(BAD))) if os.path.exists(BAD) else set()91 if "--exclude" in sys.argv:92 rep = json.load(open(sys.argv[sys.argv.index("--exclude") + 1]))93 bad |= {r["url"].lower() for r in rep["results"] if r["status"] in ("FAIL", "WARN")}94 json.dump(sorted(bad), open(BAD, "w"))95 print(f"{len(bad)} URLs excluded (validator FAIL/WARN)", file=sys.stderr)9697 other = registry_text_without_fragment()98 other_lower = other.lower()99 existing_ids = set(re.findall(r"(?m)^ - id: ([a-z0-9-]+)\s*$", other))100 existing_repos = set(m.lower() for m in re.findall(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", other))101102 lines = [HEADER.rstrip("\n"), "sources:"]103 seen_urls: set[str] = set()104 seen_ids: set[str] = set()105 n_sensors = 0106 skipped = []107 problems: list[str] = []108 # Merge duplicate `extend` entries (an org may be extended from several spec sections).109 merged: list = []110 by_id: dict = {}111 for src in SPEC:112 if isinstance(src, str) or src["id"] not in by_id:113 merged.append(src)114 if not isinstance(src, str):115 by_id[src["id"]] = src116 continue117 first = by_id[src["id"]]118 if not (first.get("extend") and src.get("extend")):119 sys.exit(f"duplicate id in spec: {src['id']}")120 for k in ("repos", "feeds", "products", "aliases"):121 first[k] = list(first.get(k, [])) + list(src.get(k, []))122 for src in merged:123 if isinstance(src, str): # section comment124 lines.append(f" # ───────────── {src} ─────────────")125 continue126 sid = src["id"]127 extend = src.get("extend", False)128 if extend and sid not in existing_ids:129 problems.append(f"extend of unknown id: {sid}"); continue130 if not extend and sid in existing_ids:131 problems.append(f"id collision with another file: {sid}"); continue132 if sid in seen_ids:133 sys.exit(f"duplicate id in spec: {sid}")134 seen_ids.add(sid)135 sensors = []136 tier = src.get("tier", "C")137 if sid not in KEEP_IDS:138 skipped.append(f"(not kept) {sid}"); continue139 cap = REPO_CAPS.get(sid, EXTEND_REPO_CAP if extend else DEFAULT_REPO_CAP)140 repos = src.get("repos", [])141 if len(repos) > cap:142 skipped.append(f"(cap {cap}) {sid}: dropped {len(repos) - cap} repos")143 repos = repos[:cap]144 for rep in repos:145 if isinstance(rep, str):146 repo, kind, label, rtier = rep, "releases", None, None147 else:148 repo = rep[0]149 kind = rep[1] if len(rep) > 1 and rep[1] else "releases"150 label = rep[2] if len(rep) > 2 else None151 rtier = rep[3] if len(rep) > 3 else None152 if repo.lower() in existing_repos:153 skipped.append(repo)154 continue155 if repo.lower() in flips and kind == "releases":156 kind = "tags"157 url = f"https://github.com/{repo}/{kind}.atom"158 if url.lower() in seen_urls or url.lower() in bad:159 continue160 seen_urls.add(url.lower())161 name = q(label or f"{repo.split('/')[1]} {kind}")162 sensors.append(f' - {{ name: {name}, url: "{url}", type: GITHUB_RELEASE, connector: github, tier: {rtier or tier}, config: {{ repo: {repo}, kind: {kind} }} }}')163 for fd in src.get("feeds", []):164 name, url = fd[0], fd[1]165 ftype = fd[2] if len(fd) > 2 else "RSS"166 ftier = fd[3] if len(fd) > 3 else tier167 if url.lower() in other_lower or url.lower() in seen_urls or url.lower() in bad:168 skipped.append(url)169 continue170 seen_urls.add(url.lower())171 sensors.append(f' - {{ name: {q(name)}, url: "{url}", type: {ftype}, connector: rss, tier: {ftier} }}')172 if not sensors:173 skipped.append(f"(no sensors) {sid}")174 continue175 n_sensors += len(sensors)176 lines.append(f" - id: {sid}")177 if extend:178 lines.append(" extend: true")179 else:180 lines.append(f" name: {q(src['name'])}")181 lines.append(f" domain: {src['domain']}")182 if src.get("homepage"):183 lines.append(f" homepage: {src['homepage']}")184 lines.append(f" categories: [{', '.join(src.get('categories', ['open-source', 'developer']))}]")185 lines.append(f" tier: {tier}")186 if src.get("weight"):187 lines.append(f" weight: {src['weight']}")188 if src.get("aliases"):189 lines.append(f" aliases: [{', '.join(q(a) for a in src['aliases'])}]")190 if src.get("products"):191 lines.append(" products:")192 for p in src["products"]:193 al = f", aliases: [{', '.join(q(a) for a in p[2])}]" if len(p) > 2 and p[2] else ""194 lines.append(f" - {{ name: {q(p[0])}, type: {p[1] if len(p) > 1 else 'software'}{al} }}")195 if not extend:196 lines.append(" discover: { rss: true }")197 if src.get("llm") is False:198 lines.append(" llm: false")199 if src.get("country"):200 lines.append(f' country: "{src["country"]}"') # quoted: YAML 1.1 reads NO (Norway) as a boolean201 if src.get("notes"):202 lines.append(f' notes: "{src["notes"]}"')203 lines.append(" sensors:")204 lines.extend(sensors)205 if problems:206 sys.exit("\n".join(problems))207 open(OUT, "w", encoding="utf-8").write("\n".join(lines) + "\n")208 print(f"wrote {OUT}: {len(seen_ids)} sources, {n_sensors} sensors; skipped {len(skipped)} already-covered", file=sys.stderr)209 for s in skipped:210 print(" skip", s, file=sys.stderr)211212213if __name__ == "__main__":214 main()215