#!/usr/bin/env python3
"""Generate config/sources.d/57-packages-top.yaml — `package` connector sensors for the most downloaded packages
of the main software registries, following the conventions of 31-packages.yaml (one sensor per (registry,
package); registries that already exist as sources are `extend: true`d; new registry sources are declared with
`llm: false`; tier D everywhere — version streams are a firehose, heuristics only).

Lists (public, reproducible — the retrieval date is written in the fragment header):
  PyPI       https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json   (monthly top 15 000 by downloads)
  npm        https://github.com/tristan-f-r/npm-rank/releases/download/latest/raw.json (top 10 000 by downloads,
             ordered by popularity; successor of anvaka/npmrank — sample/topDependents.md is gone)
  crates.io  https://crates.io/api/v1/crates?sort=downloads&per_page=100&page=1..3   (User-Agent required)
  RubyGems   https://bestgems.org/total?page=1..5  (total downloads ranking, 20 gems per page; rubygems.org's
             /api/v1/downloads/all.json only returns the 50 most downloaded *versions*)
  NuGet      https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc
  Docker Hub https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count  (official images;
             `-pull_count` sorts ascending on the current API and page 2 needs a login)
  Go         GitHub search `language:go stars:>5000` sorted by stars (pages 1–3) → module path read from the
             repository's go.mod (index.golang.org is a chronological feed, not a ranking); repositories whose
             go.mod has no domain-qualified module path are dropped.

Usage: python3 scripts/gen-packages-top.py [--cache /tmp/pkgtop] [--out config/sources.d/57-packages-top.yaml]
Packages already monitored anywhere in config/ (any `connector: package` sensor) are skipped. Afterwards run the
registry validator and prune everything that is not OK:
  node node_modules/tsx/dist/cli.mjs apps/engine/src/validate.ts config/sources.d/57-packages-top.yaml --concurrency 8 --json /tmp/pkg-report.json --quiet
  python3 scripts/prune-fragment.py config/sources.d/57-packages-top.yaml /tmp/pkg-report.json --drop-warn --keep-empty
"""
from __future__ import annotations

import argparse
import concurrent.futures as cf
import datetime as dt
import json
import os
import pathlib
import re
import subprocess
import sys
import urllib.request

ROOT = pathlib.Path(__file__).resolve().parents[1]
UA = "WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)"
LIMITS = {"pypi": 1200, "npm": 1200, "crates": 300, "rubygems": 100, "nuget": 100, "dockerhub": 100, "goproxy": 150}

RESERVED = {"y", "n", "yes", "no", "on", "off", "true", "false", "null", "~"}


def fetch(url: str, cache: pathlib.Path, name: str, headers: dict[str, str] | None = None) -> bytes:
    p = cache / name
    if p.exists() and p.stat().st_size > 0:
        return p.read_bytes()
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json, text/html;q=0.8, */*;q=0.5", **(headers or {})})
    with urllib.request.urlopen(req, timeout=60) as r:
        data = r.read()
    p.write_bytes(data)
    return data


def yq(s: str, allow_space: bool = False) -> str:
    """Quote a YAML flow scalar when it is not a safe plain scalar (`?`, `@…`, numbers, `...`, booleans…)."""
    pat = r"[A-Za-z][A-Za-z0-9 _.@+/-]*" if allow_space else r"[A-Za-z_][A-Za-z0-9_.+/-]*"
    if re.fullmatch(pat, s) and s.lower() not in RESERVED and not re.fullmatch(r"[0-9.eE+-]+", s) and not s.endswith(".") and "..." not in s:
        return s
    return json.dumps(s)


def slug(s: str) -> str:
    return re.sub(r"^-+|-+$", "", re.sub(r"[^a-z0-9]+", "-", s.lower()))[:80]


def existing_packages() -> set[tuple[str, str]]:
    seen: set[tuple[str, str]] = set()
    rx = re.compile(r"connector:\s*package.*?registry:\s*([a-z]+),\s*name:\s*\"?([^,\"}\s]+)")
    for f in [ROOT / "config" / "sources.yaml", *sorted((ROOT / "config" / "sources.d").glob("*.yaml"))]:
        if f.name == "57-packages-top.yaml":
            continue
        for m in rx.finditer(f.read_text()):
            seen.add((m.group(1), m.group(2).lower()))
    return seen


# ── list loaders ─────────────────────────────────────────────────────────────────────────────────────────────

def list_pypi(cache: pathlib.Path) -> list[str]:
    d = json.loads(fetch("https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json", cache, "pypi.json"))
    return [r["project"] for r in d["rows"]]


def list_npm(cache: pathlib.Path) -> list[str]:
    d = json.loads(fetch("https://github.com/tristan-f-r/npm-rank/releases/download/latest/raw.json", cache, "npm-rank.json"))
    return [r["name"] for r in d]


def list_crates(cache: pathlib.Path) -> list[str]:
    out: list[str] = []
    for p in (1, 2, 3):
        d = json.loads(fetch(f"https://crates.io/api/v1/crates?sort=downloads&per_page=100&page={p}", cache, f"crates-{p}.json"))
        out += [c["id"] for c in d["crates"]]
    return out


def list_rubygems(cache: pathlib.Path) -> list[str]:
    out: list[str] = []
    for p in range(1, 6):
        h = fetch(f"https://bestgems.org/total?page={p}", cache, f"bestgems-{p}.html").decode("utf8", "replace")
        for m in re.finditer(r'href="/gems/([A-Za-z0-9_.-]+)"', h):
            if m.group(1) not in out:
                out.append(m.group(1))
    return out


def list_nuget(cache: pathlib.Path) -> list[str]:
    d = json.loads(fetch("https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc", cache, "nuget.json"))
    return [x["id"] for x in d["data"]]


def list_dockerhub(cache: pathlib.Path) -> list[str]:
    d = json.loads(fetch("https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count", cache, "dockerhub.json"))
    rows = sorted(d["results"], key=lambda x: -(x.get("pull_count") or 0))
    return [x["name"] for x in rows]


def list_go(cache: pathlib.Path) -> list[str]:
    items: list[dict] = []
    for p in (1, 2, 3):
        d = json.loads(fetch(f"https://api.github.com/search/repositories?q=language:go+stars:>5000&sort=stars&order=desc&per_page=100&page={p}", cache, f"gh-go-{p}.json", {"Accept": "application/vnd.github+json"}))
        items += d.get("items", [])
    modfile = cache / "go-modules.json"
    if modfile.exists():
        mods = {r["repo"]: r["module"] for r in json.loads(modfile.read_text())}
    else:
        def one(it: dict) -> tuple[str, str | None]:
            fn, br = it["full_name"], it.get("default_branch", "HEAD")
            try:
                txt = subprocess.run(["curl", "-sL", "--max-time", "20", f"https://raw.githubusercontent.com/{fn}/{br}/go.mod"], capture_output=True, text=True, timeout=30).stdout
            except Exception:
                return fn, None
            m = re.search(r"^module\s+(\S+)", txt, re.M)
            return fn, m.group(1) if m else None
        with cf.ThreadPoolExecutor(16) as ex:
            res = list(ex.map(one, items))
        mods = dict(res)
        modfile.write_text(json.dumps([{"repo": r, "module": m, "stars": i["stargazers_count"]} for (r, m), i in zip(res, items)]))
    out: list[str] = []
    for it in items:
        m = mods.get(it["full_name"])
        if not m or "." not in m.split("/")[0] or len(m.split("/")) < 2:
            continue  # `module ragflow`, `gitea.dev`… not a fetchable module path
        if m not in out:
            out.append(m)
    return out


# ── emitters ─────────────────────────────────────────────────────────────────────────────────────────────────

def sensor(registry: str, name: str) -> str:
    label = {"pypi": "pypi", "npm": "npm", "crates": "crates", "rubygems": "rubygems", "nuget": "nuget", "dockerhub": "docker hub", "goproxy": "go"}[registry]
    page = {
        "pypi": f"https://pypi.org/project/{name}/",
        "npm": f"https://www.npmjs.com/package/{name}",
        "crates": f"https://crates.io/crates/{name}",
        "rubygems": f"https://rubygems.org/gems/{name}",
        "nuget": f"https://www.nuget.org/packages/{name}",
        "dockerhub": f"https://hub.docker.com/_/{name}",
        "goproxy": f"https://pkg.go.dev/{name}",
    }[registry]
    cfg_name = f"library/{name}" if registry == "dockerhub" else name
    return f'      - {{ name: {yq(f"{label} {name}", True)}, url: "{page}", type: REST_API, connector: package, tier: D, config: {{ registry: {registry}, name: {yq(cfg_name)}, maxItems: 30 }} }}'


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--cache", default="/tmp/pkgtop")
    ap.add_argument("--out", default=str(ROOT / "config" / "sources.d" / "57-packages-top.yaml"))
    a = ap.parse_args()
    cache = pathlib.Path(a.cache)
    cache.mkdir(parents=True, exist_ok=True)
    skip = existing_packages()
    today = dt.date.today().isoformat()

    lists = {"pypi": list_pypi(cache), "npm": list_npm(cache), "crates": list_crates(cache), "rubygems": list_rubygems(cache), "nuget": list_nuget(cache), "dockerhub": list_dockerhub(cache), "goproxy": list_go(cache)}
    chosen: dict[str, list[str]] = {}
    skipped: dict[str, int] = {}
    for reg, names in lists.items():
        out: list[str] = []
        slugs: set[str] = set()
        skipped[reg] = 0
        for n in names:
            key = (reg, (f"library/{n}" if reg == "dockerhub" else n).lower())
            if key in skip:
                skipped[reg] += 1
                continue
            s = slug(f"{reg} {n}")
            if s in slugs:
                continue  # `foo-bar` vs `foo_bar` would collide on the derived sensor id
            slugs.add(s)
            out.append(n)
            if len(out) >= LIMITS[reg]:
                break
        chosen[reg] = out

    L: list[str] = []
    L.append(f"# config/sources.d/57-packages-top.yaml — the most downloaded packages of the main software registries")
    L.append(f"# ({today}, `package` connector, tier D, heuristics only): the version streams of the software supply chain")
    L.append("# beyond the ~100 packages attached to their project entities in 31-packages.yaml. Sensors are attached to the")
    L.append("# registry's own organization (npm Registry, PyPI, Docker Hub → docker, Go module proxy → go are `extend: true`d;")
    L.append("# crates.io, RubyGems.org and NuGet Gallery are declared here with `llm: false` — the extended sources keep the")
    L.append("# `llm` of their declaring file). Every sensor was fetched and parsed by `apps/engine/src/validate.ts` before")
    L.append("# being written (packages returning WARN/FAIL were pruned). Generated by `scripts/gen-packages-top.py`; lists:")
    L.append(f"#   PyPI top {LIMITS['pypi']} by downloads — https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json")
    L.append(f"#   npm top {LIMITS['npm']} by downloads — https://github.com/tristan-f-r/npm-rank (releases/download/latest/raw.json)")
    L.append(f"#   crates.io top {LIMITS['crates']} — https://crates.io/api/v1/crates?sort=downloads&per_page=100&page=1..3")
    L.append(f"#   RubyGems top {LIMITS['rubygems']} total downloads — https://bestgems.org/total?page=1..5")
    L.append(f"#   NuGet top {LIMITS['nuget']} — https://azuresearch-usnc.nuget.org/query?take=100&sortBy=totalDownloads-desc")
    L.append(f"#   Docker Hub top {LIMITS['dockerhub']} official images — https://hub.docker.com/v2/repositories/library/?page_size=100&ordering=pull_count")
    L.append(f"#   Go top {LIMITS['goproxy']} modules — GitHub search language:go stars:>5000 sorted by stars, module path from go.mod")
    L.append("# Packages already monitored elsewhere in config/ were skipped: " + ", ".join(f"{k} {v}" for k, v in skipped.items() if v) + ".")
    L.append("sources:")

    L.append("  # ── Python (PyPI) ──")
    L.append("  - id: pypi")
    L.append("    extend: true")
    L.append("    categories: [packages]")
    L.append("    sensors:")
    L += [sensor("pypi", n) for n in chosen["pypi"]]

    L.append("  # ── JavaScript / TypeScript (npm) ──")
    L.append("  - id: npm")
    L.append("    extend: true")
    L.append("    categories: [packages]")
    L.append("    sensors:")
    L += [sensor("npm", n) for n in chosen["npm"]]

    L.append("  # ── Rust (crates.io) ──")
    L.append("  - id: crates-io")
    L.append("    name: crates.io")
    L.append("    domain: crates.io")
    L.append("    homepage: https://crates.io")
    L.append("    categories: [packages, developer, infrastructure, open-source]")
    L.append("    tier: D")
    L.append("    weight: 1.2")
    L.append("    country: US")
    L.append("    language: en")
    L.append("    aliases: [crates.io, crates, cargo registry, rust package registry]")
    L.append("    discover: { rss: true, status: true }")
    L.append("    llm: false")
    L.append('    notes: "The Rust Foundation operates crates.io; the API asks for a User-Agent and ~1 request/s (validate at --concurrency 1)."')
    L.append("    sensors:")
    L += [sensor("crates", n) for n in chosen["crates"]]

    L.append("  # ── Ruby (RubyGems) ──")
    L.append("  - id: rubygems")
    L.append("    name: RubyGems.org")
    L.append("    domain: rubygems.org")
    L.append("    homepage: https://rubygems.org")
    L.append("    categories: [packages, developer, infrastructure, open-source]")
    L.append("    tier: D")
    L.append("    weight: 1.1")
    L.append("    country: US")
    L.append("    language: en")
    L.append("    aliases: [rubygems, rubygems.org, ruby central, gem registry]")
    L.append("    discover: { rss: true, status: true }")
    L.append("    llm: false")
    L.append("    sensors:")
    L += [sensor("rubygems", n) for n in chosen["rubygems"]]

    L.append("  # ── .NET (NuGet) ──")
    L.append("  - id: nuget")
    L.append("    name: NuGet Gallery")
    L.append("    domain: nuget.org")
    L.append("    homepage: https://www.nuget.org")
    L.append("    categories: [packages, developer, infrastructure, open-source]")
    L.append("    tier: D")
    L.append("    weight: 1.1")
    L.append("    country: US")
    L.append("    language: en")
    L.append("    aliases: [nuget, nuget.org, nuget gallery, .net package registry]")
    L.append("    discover: { rss: true, status: true }")
    L.append("    llm: false")
    L.append("    sensors:")
    L += [sensor("nuget", n) for n in chosen["nuget"]]

    L.append("  # ── Containers (Docker Hub official images) ──")
    L.append("  - id: docker")
    L.append("    extend: true")
    L.append("    categories: [packages]")
    L.append("    sensors:")
    L += [sensor("dockerhub", n) for n in chosen["dockerhub"]]

    L.append("  # ── Go (module proxy) ──")
    L.append("  - id: go")
    L.append("    extend: true")
    L.append("    categories: [packages]")
    L.append("    sensors:")
    L += [sensor("goproxy", n) for n in chosen["goproxy"]]

    pathlib.Path(a.out).write_text("\n".join(L) + "\n")
    total = sum(len(v) for v in chosen.values())
    print(f"wrote {a.out}: {total} sensors — " + ", ".join(f"{k} {len(v)} (skipped {skipped[k]} existing, list {len(lists[k])})" for k, v in chosen.items()), file=sys.stderr)


if __name__ == "__main__":
    main()