TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";45/**6 * Package-registry connector — one connector for the version streams of the main software7 * registries. Each registry has its own JSON (or text) shape; all are normalized to a keyed list8 * of versions (`key = version`) plus a `latest` item whose `version` field changes on every9 * release, so the pipeline sees "new version published" and "latest tag moved" as list events.10 *11 * Config: { registry: npm | pypi | crates | rubygems | nuget | packagist | hex | goproxy | homebrew | dockerhub,12 * name: "react" | "requests" | "serde" | "rails" | "Newtonsoft.Json" | "laravel/framework" | "phoenix"13 * | "github.com/gin-gonic/gin" | "node" | "library/nginx",14 * maxItems?: 50 }15 * The sensor `url` may be anything (kept for display); the API URL is derived from registry + name16 * unless `config.url` is provided.17 */18export type Registry = "npm" | "pypi" | "crates" | "rubygems" | "nuget" | "packagist" | "hex" | "goproxy" | "homebrew" | "dockerhub";1920interface PackageConfig {21 registry?: Registry;22 name?: string;23 url?: string;24 maxItems?: number;25}2627interface VersionItem {28 key: string;29 [k: string]: unknown;30 version: string;31 title: string;32 url: string;33 publishedAt: string | null;34 summary: string;35}3637export function registryApiUrl(registry: Registry, name: string): string {38 const enc = (s: string): string => s.split("/").map(encodeURIComponent).join("/");39 switch (registry) {40 case "npm":41 return `https://registry.npmjs.org/${name.startsWith("@") ? "@" + encodeURIComponent(name.slice(1)) : encodeURIComponent(name)}`;42 case "pypi":43 return `https://pypi.org/pypi/${encodeURIComponent(name)}/json`;44 case "crates":45 return `https://crates.io/api/v1/crates/${encodeURIComponent(name)}`;46 case "rubygems":47 return `https://rubygems.org/api/v1/versions/${encodeURIComponent(name)}.json`;48 case "nuget":49 return `https://api.nuget.org/v3-flatcontainer/${encodeURIComponent(name.toLowerCase())}/index.json`;50 case "packagist":51 return `https://repo.packagist.org/p2/${enc(name)}.json`;52 case "hex":53 return `https://hex.pm/api/packages/${encodeURIComponent(name)}`;54 case "goproxy":55 return `https://proxy.golang.org/${enc(name.toLowerCase())}/@v/list`;56 case "homebrew":57 return `https://formulae.brew.sh/api/formula/${encodeURIComponent(name)}.json`;58 case "dockerhub":59 return `https://hub.docker.com/v2/repositories/${enc(name.includes("/") ? name : `library/${name}`)}/tags?page_size=50&ordering=last_updated`;60 }61}6263export function registryPageUrl(registry: Registry, name: string, version?: string): string {64 switch (registry) {65 case "npm":66 return `https://www.npmjs.com/package/${name}${version ? `/v/${version}` : ""}`;67 case "pypi":68 return `https://pypi.org/project/${name}/${version ? version + "/" : ""}`;69 case "crates":70 return `https://crates.io/crates/${name}${version ? `/${version}` : ""}`;71 case "rubygems":72 return `https://rubygems.org/gems/${name}${version ? `/versions/${version}` : ""}`;73 case "nuget":74 return `https://www.nuget.org/packages/${name}${version ? `/${version}` : ""}`;75 case "packagist":76 return `https://packagist.org/packages/${name}${version ? `#${version}` : ""}`;77 case "hex":78 return `https://hex.pm/packages/${name}${version ? `/${version}` : ""}`;79 case "goproxy":80 return `https://pkg.go.dev/${name}${version ? `@${version}` : ""}`;81 case "homebrew":82 return `https://formulae.brew.sh/formula/${name}`;83 case "dockerhub":84 return name.startsWith("library/") || !name.includes("/") ? `https://hub.docker.com/_/${name.replace(/^library\//, "")}${version ? `/tags?name=${version}` : ""}` : `https://hub.docker.com/r/${name}${version ? `/tags?name=${version}` : ""}`;85 }86}8788const PRERELEASE = /(alpha|beta|rc|canary|next|dev|preview|snapshot|nightly|experimental|insiders|pre)/i;8990export class PackageConnector implements WebSensorConnector {91 mode = "list" as const;92 metadata(): ConnectorMetadata {93 return { key: "package", name: "Package registry", sensorTypes: ["REST_API", "JSON"], description: "Version streams of npm, PyPI, crates.io, RubyGems, NuGet, Packagist, Hex, Go proxy, Homebrew and Docker Hub", version: "1.0.0" };94 }9596 private resolve(endpoint: SensorEndpoint): { registry: Registry; name: string; url: string } {97 const cfg = endpoint.config as PackageConfig;98 if (!cfg.registry || !cfg.name) throw new NormalizeError("bad_config", "package connector needs config.registry and config.name");99 return { registry: cfg.registry, name: cfg.name, url: cfg.url ?? registryApiUrl(cfg.registry, cfg.name) };100 }101102 async fetch(endpoint: SensorEndpoint): Promise<Observation> {103 let target: { registry: Registry; name: string; url: string };104 try {105 target = this.resolve(endpoint);106 } catch (e) {107 return { sensorId: endpoint.id, url: endpoint.url, fetchedAt: new Date(), notModified: false, error: { code: "bad_config", message: (e as Error).message }, meta: { status: 0, url: endpoint.url, finalUrl: endpoint.url, contentType: null, contentLength: 0, etag: null, lastModified: null, durationMs: 0, redirects: 0, method: "GET", headers: {} } };108 }109 const accept = target.registry === "npm" ? "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8" : target.registry === "goproxy" ? "text/plain, */*;q=0.5" : "application/json, */*;q=0.5";110 const obs = await httpFetchWithRetry(endpoint.id, target.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept, headers: target.registry === "crates" ? { "user-agent": "WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)" } : undefined, timeoutMs: 40_000, maxBytes: 60 * 1024 * 1024 });111 return { ...obs, url: endpoint.url };112 }113114 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {115 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");116 const { registry, name } = this.resolve(endpoint);117 const cfg = endpoint.config as PackageConfig;118 const text = obs.body.toString("utf8");119 let versions: VersionItem[];120 let latest: string | undefined;121 let description = "";122 const page = (v?: string): string => registryPageUrl(registry, name, v);123 const item = (version: string, publishedAt: string | null, extra: Record<string, unknown> = {}): VersionItem => ({ key: version, version, title: `${name} ${version}`, url: page(version), publishedAt: publishedAt && !Number.isNaN(new Date(publishedAt).getTime()) ? new Date(publishedAt).toISOString() : null, summary: "", prerelease: PRERELEASE.test(version), ...extra });124125 if (registry === "goproxy") {126 versions = text127 .split(/\r?\n/)128 .map((s) => s.trim())129 .filter(Boolean)130 .map((v) => item(v, null));131 versions.sort((a, b) => semverCompare(b.version, a.version));132 latest = versions.find((v) => !v.prerelease)?.version ?? versions[0]?.version;133 } else {134 let j: unknown;135 try {136 j = JSON.parse(text);137 } catch {138 throw new NormalizeError("bad_json", `${registry} response is not JSON`);139 }140 const o = j as Record<string, unknown>;141 switch (registry) {142 case "npm": {143 const time = (o.time ?? {}) as Record<string, string>;144 const tags = (o["dist-tags"] ?? {}) as Record<string, string>;145 description = String(o.description ?? "");146 versions = Object.keys((o.versions ?? {}) as object).map((v) => item(v, time[v] ?? null));147 if (!versions.length) versions = Object.keys(time).filter((k) => k !== "created" && k !== "modified").map((v) => item(v, time[v] ?? null));148 latest = tags.latest;149 versions.sort((a, b) => (b.publishedAt ?? "").localeCompare(a.publishedAt ?? ""));150 break;151 }152 case "pypi": {153 const info = (o.info ?? {}) as Record<string, unknown>;154 description = String(info.summary ?? "");155 latest = String(info.version ?? "");156 const rel = (o.releases ?? {}) as Record<string, { upload_time_iso_8601?: string; yanked?: boolean }[]>;157 versions = Object.entries(rel).map(([v, files]) => item(v, files[0]?.upload_time_iso_8601 ?? null, { yanked: files.some((f) => f.yanked) }));158 versions.sort((a, b) => (b.publishedAt ?? "").localeCompare(a.publishedAt ?? ""));159 break;160 }161 case "crates": {162 const crate = (o.crate ?? {}) as Record<string, unknown>;163 description = String(crate.description ?? "");164 latest = String(crate.max_stable_version ?? crate.max_version ?? "");165 versions = ((o.versions ?? []) as Record<string, unknown>[]).map((v) => item(String(v.num), (v.created_at as string) ?? null, { yanked: Boolean(v.yanked) }));166 break;167 }168 case "rubygems": {169 versions = ((Array.isArray(j) ? j : []) as Record<string, unknown>[]).map((v) => item(String(v.number), (v.created_at as string) ?? null, { prerelease: Boolean(v.prerelease) }));170 latest = versions.find((v) => !v.prerelease)?.version;171 description = String(((Array.isArray(j) ? j[0] : {}) as Record<string, unknown>)?.summary ?? "");172 break;173 }174 case "nuget": {175 versions = ((o.versions ?? []) as string[]).map((v) => item(v, null)).reverse();176 latest = versions.find((v) => !v.prerelease)?.version ?? versions[0]?.version;177 break;178 }179 case "packagist": {180 const pk = ((o.packages ?? {}) as Record<string, Record<string, unknown>[]>)[name] ?? [];181 versions = pk.filter((v) => typeof v.version === "string").map((v) => item(String(v.version), (v.time as string) ?? null));182 latest = versions.find((v) => !v.prerelease && !/dev/.test(v.version))?.version;183 description = String(pk[0]?.description ?? "");184 break;185 }186 case "hex": {187 const meta = (o.meta ?? {}) as Record<string, unknown>;188 description = String(meta.description ?? "");189 versions = ((o.releases ?? []) as Record<string, unknown>[]).map((v) => item(String(v.version), (v.inserted_at as string) ?? null));190 latest = String(((o.latest_stable_version ?? o.latest_version) as string) ?? versions[0]?.version ?? "");191 break;192 }193 case "homebrew": {194 const vs = (o.versions ?? {}) as Record<string, unknown>;195 description = String(o.desc ?? "");196 latest = String(vs.stable ?? "");197 versions = latest ? [item(latest, null, { revision: o.revision ?? 0, bottle: vs.bottle })] : [];198 break;199 }200 case "dockerhub": {201 versions = ((o.results ?? []) as Record<string, unknown>[]).map((t) => item(String(t.name), (t.last_updated as string) ?? null, { digest: String(t.digest ?? "").slice(0, 32) }));202 latest = versions.find((v) => v.version === "latest") ? "latest" : versions[0]?.version;203 break;204 }205 default:206 throw new NormalizeError("bad_config", `unsupported registry ${String(registry)}`);207 }208 }209210 const max = cfg.maxItems ?? 50;211 const items: { key: string; [k: string]: unknown }[] = versions.slice(0, max);212 const latestItem = versions.find((v) => v.version === latest);213 items.unshift({ key: "latest", kind: "latest", version: latest ?? "", title: `${name} latest → ${latest ?? "?"}`, url: page(latest), summary: description.slice(0, 400), publishedAt: latestItem?.publishedAt ?? null, digest: (latestItem as { digest?: string } | undefined)?.digest });214 // For Docker Hub, tag content changes without a new tag name (same key, new digest) → compare digest too.215 const compareFields = registry === "dockerhub" ? ["version", "digest"] : registry === "homebrew" ? ["version", "revision"] : ["version"];216 const canonical = items.map((i) => `${i.key}\t${compareFields.map((f) => String(i[f] ?? "")).join("\t")}`).join("\n");217 const newest = items.map((i) => i.publishedAt as string | null).filter((x): x is string => Boolean(x)).sort().at(-1);218 return {219 mode: "list",220 items,221 compareFields,222 title: `${registry}:${name}`,223 rawHash: sha256(text),224 canonicalHash: sha256(canonical),225 semanticHash: simhash(items.map((i) => String(i.title)).join("\n")),226 publishedAt: newest ? new Date(newest) : null,227 extractionConfidence: 1,228 extra: { registry, name, latest, versionCount: versions.length, description: description.slice(0, 200) },229 };230 }231}232233/** Loose semver comparison (handles v-prefix, prerelease after numeric parts). */234export function semverCompare(a: string, b: string): number {235 const pa = a.replace(/^v/, "").split(/[.+-]/);236 const pb = b.replace(/^v/, "").split(/[.+-]/);237 for (let i = 0; i < Math.max(pa.length, pb.length); i++) {238 const x = pa[i];239 const y = pb[i];240 if (x === undefined) return -1;241 if (y === undefined) return 1;242 const nx = Number(x);243 const ny = Number(y);244 if (!Number.isNaN(nx) && !Number.isNaN(ny)) {245 if (nx !== ny) return nx - ny;246 } else if (x !== y) return x.localeCompare(y);247 }248 return 0;249}250