spb/websensor

1import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";4import { parseFeed } from "./rss";56/**7 * GitHub connector. Prefers the unauthenticated Atom feeds (`/releases.atom`, `/tags.atom`,8 * `/commits/<branch>.atom`) which are not subject to the 60 req/h REST limit; uses the REST9 * API (with optional GITHUB_TOKEN) only for `security advisories`.10 * Config: { repo: "owner/name", kind: "releases" | "tags" | "commits" | "advisories", branch?: string }11 */12export class GitHubConnector implements WebSensorConnector {13  mode = "list" as const;14  metadata(): ConnectorMetadata {15    return { key: "github", name: "GitHub", sensorTypes: ["GITHUB_RELEASE", "GITHUB_REPO"], description: "Releases, tags, commits (Atom) and security advisories (REST)", version: "1.0.0" };16  }17  private urlFor(endpoint: SensorEndpoint): { url: string; api: boolean } {18    const cfg = endpoint.config as { repo?: string; kind?: string; branch?: string };19    if (!cfg.repo) return { url: endpoint.url, api: /api\.github\.com/.test(endpoint.url) };20    switch (cfg.kind ?? "releases") {21      case "tags":22        return { url: `https://github.com/${cfg.repo}/tags.atom`, api: false };23      case "commits":24        return { url: `https://github.com/${cfg.repo}/commits/${cfg.branch ?? "main"}.atom`, api: false };25      case "advisories":26        return { url: `https://api.github.com/repos/${cfg.repo}/security-advisories?per_page=30`, api: true };27      default:28        return { url: `https://github.com/${cfg.repo}/releases.atom`, api: false };29    }30  }31  async fetch(endpoint: SensorEndpoint): Promise<Observation> {32    const { url, api } = this.urlFor(endpoint);33    const headers: Record<string, string> = {};34    if (api) {35      headers.accept = "application/vnd.github+json";36      headers["x-github-api-version"] = "2022-11-28";37      if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`;38    }39    const obs = await httpFetchWithRetry(endpoint.id, url, { etag: endpoint.etag, lastModified: endpoint.lastModified, headers, accept: api ? "application/vnd.github+json" : "application/atom+xml, application/xml;q=0.9" });40    return { ...obs, url: endpoint.url };41  }42  async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {43    if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");44    const text = obs.body.toString("utf8");45    const { api } = this.urlFor(endpoint);46    if (api) {47      let arr: Record<string, unknown>[];48      try {49        arr = JSON.parse(text) as Record<string, unknown>[];50      } catch {51        throw new NormalizeError("bad_json", "GitHub API response is not JSON");52      }53      if (!Array.isArray(arr)) throw new NormalizeError("bad_shape", String((arr as Record<string, unknown>).message ?? "Unexpected GitHub API response"));54      const items = arr.map((a) => ({ key: String(a.ghsa_id ?? a.id), title: `${String(a.ghsa_id ?? "")} ${String(a.summary ?? "")}`.trim(), url: String(a.html_url ?? ""), summary: String(a.description ?? "").slice(0, 1000), severity: String(a.severity ?? ""), cve: String(a.cve_id ?? ""), publishedAt: a.published_at ? new Date(String(a.published_at)).toISOString() : null, state: String(a.state ?? "") }));55      const canonical = items.map((i) => `${i.key}\t${i.state}\t${i.severity}`).join("\n");56      return { mode: "list", items, compareFields: ["state", "severity", "title"], rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(items.map((i) => i.title).join("\n")), publishedAt: items[0]?.publishedAt ? new Date(items[0].publishedAt) : null, extractionConfidence: 1 };57    }58    const feed = parseFeed(text, obs.meta.finalUrl);59    const items = feed.items.slice(0, 60);60    const canonical = items.map((i) => `${i.key}\t${i.title}\t${i.updatedAt ?? ""}`).join("\n");61    const newest = items.map((i) => i.publishedAt).filter((x): x is string => Boolean(x)).sort().at(-1);62    return { mode: "list", items, compareFields: ["title"], title: feed.title, rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(items.map((i) => i.title).join("\n")), publishedAt: newest ? new Date(newest) : null, extractionConfidence: 1, extra: { feedKind: feed.kind } };63  }64}65