#!/usr/bin/env python3 """Generate config/sources.d/47-open-source-long-tail.yaml from the compact spec in scripts/gen-open-source-long-tail-spec*.py (six parts, executed in order). Usage: python3 scripts/gen-open-source-long-tail.py # write the fragment python3 scripts/gen-open-source-long-tail.py --exclude /tmp/v47.json # re-generate without every URL the validator reported as FAIL or WARN (remembered in /tmp/v47-bad.json) python3 scripts/gen-open-source-long-tail.py --flip /tmp/v47.json # re-generate, turning every `releases.atom` that the validator reported as # `WARN empty list` into a `tags.atom` sensor (kind: tags); flips are remembered in # /tmp/v47-flips.json so several passes accumulate. Safety: repos / URLs already present anywhere else in config/ are skipped, and a new (non-extend) id that already exists in another file aborts the run. """ import json import os import re import sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUT = os.path.join(ROOT, "config/sources.d/47-open-source-long-tail.yaml") FLIPS = "/tmp/v47-flips.json" BAD = "/tmp/v47-bad.json" # Keep entries reviewable and the fragment within its 350–500 sensor budget: at most this many GitHub repos per # organization unless listed in REPO_CAPS; only ids in KEEP_IDS enter the fragment (the rest of the spec is kept for a later fragment). DEFAULT_REPO_CAP = 1 EXTEND_REPO_CAP = 1 REPO_CAPS = {'apache': 26, 'cncf': 30, 'python': 4, 'nodejs': 3, 'go': 3, 'rust': 3, 'java-libraries': 6, 'react': 3, 'dotnet': 3, 'kotlin': 2, 'swift': 2, 'ruby': 3, 'php': 3, 'google': 4, 'aws': 3, 'red-hat': 3, 'meta-open-source': 3, 'postgresql': 3, 'openjs': 6, 'eclipse': 3, 'microsoft': 2, 'rust-foundation': 2, 'scala': 2, 'elixir': 2, 'vue': 2, 'angular': 2, 'tailwind': 2, 'spring': 2, 'laravel': 2, 'webassembly': 3, 'openinfra': 2, 'openssf': 2, 'docker': 2, 'hashicorp': 2, 'ubuntu': 2, 'github': 2, 'gnome': 2, 'kde': 2, 'osgeo': 3, 'openapi-tools': 3, 'terraform-tooling': 2, 'freedesktop': 1} KEEP_IDS = set(""" linux-foundation numfocus fsfe openinfra bytecode-alliance webassembly openjs eclipse apache cncf openssf rust-foundation rust python astral nodejs go ruby rails php symfony laravel spring quarkus micronaut openjdk graalvm java-libraries kotlin swift elixir erlang gleam scala haskell ocaml dart flutter lua nim crystal vlang odin-lang mojo perl clojure racket julia r-project posit dotnet deno microsoft typescript expressjs fastify nestjs hono remix nuxt svelte solidjs qwik htmx tanstack vue react react-native angular tailwind shadcn-ui radix-ui mui chakra-ui ant-design storybook nx biome oxc swc rspack vite webpack eslint prettier graphql apollo-graphql hasura prisma keycloak ory authentik zitadel cypress testing-library d3 chartjs plotly threejs maplibre osgeo mermaid postgresql sqlite mongodb redis duckdb clickhouse elastic neo4j cassandra kafka rabbitmq airflow dbt redpanda kong caddy nginx questdb arangodb scylladb opensearch milvus pingcap yugabyte dragonflydb valkey influxdata trino debezium metabase prefect dagster appwrite quickwit emqx zeromq camunda openresty varnish swagger openapi-tools grafana sentry datadog posthog victoriametrics netdata uptime-kuma signoz zipkin plausible matomo zabbix nagios icinga podman opencontainers docker firecracker lima libvirt sidero-labs k0s rancher linuxcontainers ubuntu hashicorp ansible pulumi saltproject terragrunt atlantis sops github gitlab dagger flatpak renovate wireguard openvpn gnu obsidian alacritty wezterm kitty ghostty tmux fish-shell zsh starship helix git gitea forgejo guix nixos mozilla homebrew neovim vim zed jetbrains gimp gnome kde audacity keepassxc nextcloud syncthing borgbackup jellyfin immich home-assistant openhab openwrt netgate opnsense pi-hole adguard-dns telegram jitsi mattermost matrix-ecosystem onlyoffice wikimedia zotero anki ffmpeg imagemagick freecad arduino espressif micropython raspberry-pi zephyr freertos khronos winehq libretro wordpress woocommerce payload-cms directus keystatic medusa saleor magento prestashop shopware typo3 hugo latex-project discourse mastodon bluesky-atproto ionic expo scikit-learn xgboost pandas polars numpy scipy matplotlib ray dask onnx lightning-ai explosion opencv intel nvidia huggingface pytorch tensorflow jax linux-kernel systemd freedesktop hyprland sway xfce linux-mint rocky-linux almalinux void-linux freebsd openbsd netbsd haiku reactos ladybird google meta-open-source shopify stripe alibaba-cloud red-hat suse aws astro pandoc typst """.split()) def load_spec(): """Execute scripts/gen-open-source-long-tail-spec*.py in order in one namespace and concatenate SPEC, SPEC2 … SPEC6.""" here = os.path.dirname(os.path.abspath(__file__)) ns: dict = {} spec: list = [] for n in ["", "2", "3", "4", "5", "6"]: exec(compile(open(os.path.join(here, f"gen-open-source-long-tail-spec{n}.py"), encoding="utf-8").read(), f"spec{n}", "exec"), ns) spec += ns[f"SPEC{n}"] return spec, ns["HEADER"] SPEC, HEADER = load_spec() def q(v: str) -> str: """Quote a scalar for compact flow-style YAML when it contains characters that cannot start or sit in a plain scalar.""" if re.search(r'[:#{}\[\],&*!|>\'"%@`]|^[-?]|^\s|\s$', v) or v.lower() in {"yes", "no", "true", "false", "null", "on", "off"}: return json.dumps(v, ensure_ascii=False) return v def registry_text_without_fragment() -> str: parts = [] cfg = os.path.join(ROOT, "config") for dp, _, fns in os.walk(cfg): for fn in fns: p = os.path.join(dp, fn) if fn.endswith(".yaml") and os.path.abspath(p) != os.path.abspath(OUT): parts.append(open(p, encoding="utf-8").read()) return "\n".join(parts) def main() -> None: flips = set(json.load(open(FLIPS))) if os.path.exists(FLIPS) else set() if "--flip" in sys.argv: rep = json.load(open(sys.argv[sys.argv.index("--flip") + 1])) for r in rep["results"]: if r["status"] == "WARN" and r["note"].startswith("empty list") and r["url"].endswith("/releases.atom"): m = re.search(r"github\.com/([^/]+/[^/]+)/releases\.atom", r["url"]) if m: flips.add(m.group(1).lower()) json.dump(sorted(flips), open(FLIPS, "w")) print(f"{len(flips)} repos flipped to tags", file=sys.stderr) bad = set(json.load(open(BAD))) if os.path.exists(BAD) else set() if "--exclude" in sys.argv: rep = json.load(open(sys.argv[sys.argv.index("--exclude") + 1])) bad |= {r["url"].lower() for r in rep["results"] if r["status"] in ("FAIL", "WARN")} json.dump(sorted(bad), open(BAD, "w")) print(f"{len(bad)} URLs excluded (validator FAIL/WARN)", file=sys.stderr) other = registry_text_without_fragment() other_lower = other.lower() existing_ids = set(re.findall(r"(?m)^ - id: ([a-z0-9-]+)\s*$", other)) existing_repos = set(m.lower() for m in re.findall(r"github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", other)) lines = [HEADER.rstrip("\n"), "sources:"] seen_urls: set[str] = set() seen_ids: set[str] = set() n_sensors = 0 skipped = [] problems: list[str] = [] # Merge duplicate `extend` entries (an org may be extended from several spec sections). merged: list = [] by_id: dict = {} for src in SPEC: if isinstance(src, str) or src["id"] not in by_id: merged.append(src) if not isinstance(src, str): by_id[src["id"]] = src continue first = by_id[src["id"]] if not (first.get("extend") and src.get("extend")): sys.exit(f"duplicate id in spec: {src['id']}") for k in ("repos", "feeds", "products", "aliases"): first[k] = list(first.get(k, [])) + list(src.get(k, [])) for src in merged: if isinstance(src, str): # section comment lines.append(f" # ───────────── {src} ─────────────") continue sid = src["id"] extend = src.get("extend", False) if extend and sid not in existing_ids: problems.append(f"extend of unknown id: {sid}"); continue if not extend and sid in existing_ids: problems.append(f"id collision with another file: {sid}"); continue if sid in seen_ids: sys.exit(f"duplicate id in spec: {sid}") seen_ids.add(sid) sensors = [] tier = src.get("tier", "C") if sid not in KEEP_IDS: skipped.append(f"(not kept) {sid}"); continue cap = REPO_CAPS.get(sid, EXTEND_REPO_CAP if extend else DEFAULT_REPO_CAP) repos = src.get("repos", []) if len(repos) > cap: skipped.append(f"(cap {cap}) {sid}: dropped {len(repos) - cap} repos") repos = repos[:cap] for rep in repos: if isinstance(rep, str): repo, kind, label, rtier = rep, "releases", None, None else: repo = rep[0] kind = rep[1] if len(rep) > 1 and rep[1] else "releases" label = rep[2] if len(rep) > 2 else None rtier = rep[3] if len(rep) > 3 else None if repo.lower() in existing_repos: skipped.append(repo) continue if repo.lower() in flips and kind == "releases": kind = "tags" url = f"https://github.com/{repo}/{kind}.atom" if url.lower() in seen_urls or url.lower() in bad: continue seen_urls.add(url.lower()) name = q(label or f"{repo.split('/')[1]} {kind}") sensors.append(f' - {{ name: {name}, url: "{url}", type: GITHUB_RELEASE, connector: github, tier: {rtier or tier}, config: {{ repo: {repo}, kind: {kind} }} }}') for fd in src.get("feeds", []): name, url = fd[0], fd[1] ftype = fd[2] if len(fd) > 2 else "RSS" ftier = fd[3] if len(fd) > 3 else tier if url.lower() in other_lower or url.lower() in seen_urls or url.lower() in bad: skipped.append(url) continue seen_urls.add(url.lower()) sensors.append(f' - {{ name: {q(name)}, url: "{url}", type: {ftype}, connector: rss, tier: {ftier} }}') if not sensors: skipped.append(f"(no sensors) {sid}") continue n_sensors += len(sensors) lines.append(f" - id: {sid}") if extend: lines.append(" extend: true") else: lines.append(f" name: {q(src['name'])}") lines.append(f" domain: {src['domain']}") if src.get("homepage"): lines.append(f" homepage: {src['homepage']}") lines.append(f" categories: [{', '.join(src.get('categories', ['open-source', 'developer']))}]") lines.append(f" tier: {tier}") if src.get("weight"): lines.append(f" weight: {src['weight']}") if src.get("aliases"): lines.append(f" aliases: [{', '.join(q(a) for a in src['aliases'])}]") if src.get("products"): lines.append(" products:") for p in src["products"]: al = f", aliases: [{', '.join(q(a) for a in p[2])}]" if len(p) > 2 and p[2] else "" lines.append(f" - {{ name: {q(p[0])}, type: {p[1] if len(p) > 1 else 'software'}{al} }}") if not extend: lines.append(" discover: { rss: true }") if src.get("llm") is False: lines.append(" llm: false") if src.get("country"): lines.append(f' country: "{src["country"]}"') # quoted: YAML 1.1 reads NO (Norway) as a boolean if src.get("notes"): lines.append(f' notes: "{src["notes"]}"') lines.append(" sensors:") lines.extend(sensors) if problems: sys.exit("\n".join(problems)) open(OUT, "w", encoding="utf-8").write("\n".join(lines) + "\n") print(f"wrote {OUT}: {len(seen_ids)} sources, {n_sensors} sensors; skipped {len(skipped)} already-covered", file=sys.stderr) for s in skipped: print(" skip", s, file=sys.stderr) if __name__ == "__main__": main()