spb/websensor

1import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { getPath } from "./http";4import { NormalizeError, type WebSensorConnector } from "./types";56/**7 * Generic JSON API list connector — covers official public APIs that return a list of8 * records: CISA KEV, Federal Register, ClinicalTrials.gov v2, USGS GeoJSON, Have I Been9 * Pwned breaches, NVD CVE 2.0, JSON changelogs…10 *11 * Config:12 *   itemsPath: "vulnerabilities" | "results" | "studies" | "features" | ""   (dot path to the array)13 *   keyField:  "cveID" | "document_number" | "protocolSection.identificationModule.nctId" | "id"14 *   titleField, urlField, summaryField, dateField: dot paths inside each item15 *   compareFields: string[]  (fields whose change counts as an update)16 *   urlTemplate: "https://nvd.nist.gov/vuln/detail/{key}"17 *   headers: {}  (e.g. User-Agent contact required by SEC)18 *   maxItems: 20019 */20export class JsonListConnector implements WebSensorConnector {21  mode = "list" as const;22  metadata(): ConnectorMetadata {23    return { key: "jsonlist", name: "JSON API list", sensorTypes: ["REST_API", "JSON"], description: "Keyed records from an official JSON API (KEV, Federal Register, ClinicalTrials, USGS, HIBP, NVD…)", version: "1.0.0" };24  }25  async fetch(endpoint: SensorEndpoint): Promise<Observation> {26    const cfg = endpoint.config as { headers?: Record<string, string>; url?: string; timeoutMs?: number; noConditional?: boolean };27    // Support {date} placeholders for APIs that require a window (NVD).28    const url = expandUrl(cfg.url ?? endpoint.url);29    const obs = await httpFetchWithRetry(endpoint.id, url, { etag: cfg.noConditional ? null : endpoint.etag, lastModified: cfg.noConditional ? null : endpoint.lastModified, headers: cfg.headers, accept: "application/json, */*;q=0.5", timeoutMs: cfg.timeoutMs ?? 40_000, maxBytes: 40 * 1024 * 1024 });30    return { ...obs, url: endpoint.url };31  }32  async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {33    if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");34    const text = obs.body.toString("utf8");35    let root: unknown;36    try {37      root = JSON.parse(text);38    } catch {39      throw new NormalizeError("bad_json", "API response is not JSON");40    }41    const cfg = endpoint.config as { itemsPath?: string; keyField?: string; titleField?: string; urlField?: string; summaryField?: string; dateField?: string; compareFields?: string[]; urlTemplate?: string; maxItems?: number; titleTemplate?: string };42    const arr = cfg.itemsPath ? getPath(root, cfg.itemsPath) : root;43    if (!Array.isArray(arr)) throw new NormalizeError("bad_shape", `itemsPath "${cfg.itemsPath ?? ""}" is not an array`);44    const items = (arr as Record<string, unknown>[]).slice(0, cfg.maxItems ?? 300).map((it) => {45      const key = str(getPath(it, cfg.keyField ?? "id"));46      const title = cfg.titleTemplate ? fill(cfg.titleTemplate, it) : str(getPath(it, cfg.titleField ?? "title"));47      // `{key}` keeps path separators (Hugging Face `owner/model` ids); `{field.path}` fills from the record.48      const url = cfg.urlTemplate ? fill(cfg.urlTemplate.replace("{key}", key.split("/").map(encodeURIComponent).join("/")), it) : str(getPath(it, cfg.urlField ?? "url"));49      const summary = str(getPath(it, cfg.summaryField ?? "summary")).slice(0, 1200);50      const dateRaw = str(getPath(it, cfg.dateField ?? "date"));51      const d = dateRaw ? new Date(dateRaw) : null;52      const out: Record<string, unknown> & { key: string } = { key, title, url, summary, publishedAt: d && !Number.isNaN(d.getTime()) ? d.toISOString() : null };53      for (const f of cfg.compareFields ?? []) out[f] = getPath(it, f);54      return out;55    });56    const compareFields = cfg.compareFields ?? ["title"];57    const canonical = items.map((i) => `${i.key}\t${compareFields.map((f) => JSON.stringify(i[f] ?? "")).join("\t")}`).join("\n");58    const newest = items.map((i) => i.publishedAt as string | null).filter((x): x is string => Boolean(x)).sort().at(-1);59    return { mode: "list", items, compareFields, rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(items.map((i) => String(i.title)).join("\n")), publishedAt: newest ? new Date(newest) : null, extractionConfidence: 1, extra: { count: items.length } };60  }61}6263function str(v: unknown): string {64  if (v === null || v === undefined) return "";65  if (typeof v === "string") return v;66  if (typeof v === "number" || typeof v === "boolean") return String(v);67  if (Array.isArray(v)) return v.map(str).filter(Boolean).join(", ");68  if (typeof v === "object") {69    const o = v as Record<string, unknown>;70    if (typeof o.value === "string") return o.value;71    if (Array.isArray(o.descriptions)) return str((o.descriptions as Record<string, unknown>[]).find((d) => d.lang === "en")?.value ?? o.descriptions[0]);72  }73  return JSON.stringify(v).slice(0, 400);74}7576function fill(tpl: string, it: Record<string, unknown>): string {77  return tpl.replace(/\{([^}]+)\}/g, (_, p: string) => str(getPath(it, p)));78}7980/** `{now-2h}` / `{now}` placeholders → ISO-8601 (no millis, NVD-compatible). */81export function expandUrl(url: string): string {82  return url.replace(/\{now(?:-(\d+)([hmd]))?\}/g, (_, n: string | undefined, u: string | undefined) => {83    const d = new Date();84    if (n && u) {85      const ms = Number(n) * (u === "h" ? 3600e3 : u === "m" ? 60e3 : 86400e3);86      d.setTime(d.getTime() - ms);87    }88    return encodeURIComponent(d.toISOString().replace(/\.\d{3}Z$/, ".000"));89  });90}91