TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import YAML from "yaml";2import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";3import { httpFetchWithRetry } from "./fetcher";4import { NormalizeError, type WebSensorConnector } from "./types";56/**7 * OpenAPI / Swagger connector. Fetches a public API description (JSON or YAML, OpenAPI 3.x or8 * Swagger 2.0) and normalizes it to a keyed list of operations (`GET /v1/charges`) with a stable9 * fingerprint of each operation's contract (parameters, request body, response codes, deprecation,10 * summary). Adding, removing or deprecating an endpoint, or changing its schema, becomes an11 * `api_change` list event; the `info` item tracks the declared API version.12 *13 * Specs can be large (Stripe ≈ 8 MB, GitHub ≈ 13 MB) — 64 MB limit, 60 s timeout.14 * Config: { maxOperations?: 5000, pathPrefix?: "/v1" }15 */16interface Op {17 key: string;18 [k: string]: unknown;19 method: string;20 path: string;21 title: string;22 summary: string;23 url: string;24 deprecated: boolean;25 fingerprint: string;26 publishedAt: null;27}2829export function parseSpec(text: string, contentType: string | null): Record<string, unknown> {30 const t = text.trimStart();31 if (t.startsWith("{") || (contentType ?? "").includes("json")) {32 try {33 return JSON.parse(text) as Record<string, unknown>;34 } catch {35 /* fall through to YAML */36 }37 }38 try {39 const y = YAML.parse(text, { maxAliasCount: -1 }) as unknown;40 if (y && typeof y === "object") return y as Record<string, unknown>;41 } catch (e) {42 throw new NormalizeError("bad_spec", `Not JSON nor YAML: ${(e as Error).message.slice(0, 120)}`);43 }44 throw new NormalizeError("bad_spec", "Document is not an object");45}4647const METHODS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"];4849/** Stable JSON with sorted keys, descriptions/examples stripped (prose edits are not contract changes). */50function fingerprint(v: unknown): string {51 const strip = (x: unknown): unknown => {52 if (Array.isArray(x)) return x.map(strip);53 if (x && typeof x === "object") {54 const out: Record<string, unknown> = {};55 for (const k of Object.keys(x as object).sort()) {56 if (k === "description" || k === "example" || k === "examples" || k === "x-codeSamples" || k === "externalDocs" || k.startsWith("x-")) continue;57 out[k] = strip((x as Record<string, unknown>)[k]);58 }59 return out;60 }61 return x;62 };63 return sha256(JSON.stringify(strip(v))).slice(0, 16);64}6566export function extractOperations(spec: Record<string, unknown>, opts: { pathPrefix?: string; maxOperations?: number; baseUrl?: string } = {}): { ops: Op[]; info: Record<string, unknown>; version: string; kind: string } {67 const paths = (spec.paths ?? {}) as Record<string, Record<string, unknown>>;68 const info = (spec.info ?? {}) as Record<string, unknown>;69 const kind = spec.openapi ? `openapi ${String(spec.openapi)}` : spec.swagger ? `swagger ${String(spec.swagger)}` : "unknown";70 const ops: Op[] = [];71 const max = opts.maxOperations ?? 5000;72 for (const p of Object.keys(paths).sort()) {73 if (opts.pathPrefix && !p.startsWith(opts.pathPrefix)) continue;74 const item = paths[p] ?? {};75 const shared = { parameters: item.parameters };76 for (const m of METHODS) {77 const op = item[m] as Record<string, unknown> | undefined;78 if (!op) continue;79 const method = m.toUpperCase();80 const responses = Object.keys((op.responses ?? {}) as object).sort();81 const fp = fingerprint({ parameters: op.parameters, shared, requestBody: op.requestBody, responses: op.responses, security: op.security, deprecated: op.deprecated, operationId: op.operationId, summary: op.summary });82 ops.push({ key: `${method} ${p}`, method, path: p, title: `${method} ${p}${op.deprecated ? " (deprecated)" : ""}`, summary: String(op.summary ?? op.operationId ?? "").slice(0, 300), url: opts.baseUrl ?? "", deprecated: Boolean(op.deprecated), tags: (op.tags as string[] | undefined) ?? [], responses: responses.join(","), fingerprint: fp, publishedAt: null });83 if (ops.length >= max) return { ops, info, version: String(info.version ?? ""), kind };84 }85 }86 return { ops, info, version: String(info.version ?? ""), kind };87}8889export class OpenApiConnector implements WebSensorConnector {90 mode = "list" as const;91 metadata(): ConnectorMetadata {92 return { key: "openapi", name: "OpenAPI / Swagger", sensorTypes: ["JSON", "FILE"], description: "API description → operations list with contract fingerprints (added/removed/deprecated/changed endpoints, version)", version: "1.0.0" };93 }94 async fetch(endpoint: SensorEndpoint): Promise<Observation> {95 return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/json, application/yaml, text/yaml, text/plain;q=0.8, */*;q=0.5", timeoutMs: 60_000, maxBytes: 64 * 1024 * 1024 });96 }97 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {98 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");99 const text = obs.body.toString("utf8");100 const spec = parseSpec(text, obs.meta.contentType);101 if (!spec.paths || typeof spec.paths !== "object") throw new NormalizeError("bad_spec", "No `paths` object — not an OpenAPI/Swagger document");102 const cfg = endpoint.config as { pathPrefix?: string; maxOperations?: number; docsUrl?: string };103 const { ops, info, version, kind } = extractOperations(spec, { pathPrefix: cfg.pathPrefix, maxOperations: cfg.maxOperations, baseUrl: cfg.docsUrl ?? endpoint.url });104 const items: { key: string; [k: string]: unknown }[] = [{ key: "info", kind: "info", title: `${String(info.title ?? "API")} — version ${version || "?"}`, version, fingerprint: version, summary: String(info.description ?? "").slice(0, 300), url: cfg.docsUrl ?? endpoint.url, deprecated: false, publishedAt: null }, ...ops];105 const canonical = items.map((i) => `${i.key}\t${String(i.fingerprint)}\t${String(i.deprecated)}`).join("\n");106 return {107 mode: "list",108 items,109 compareFields: ["fingerprint", "deprecated"],110 title: String(info.title ?? endpoint.name),111 rawHash: sha256(text),112 canonicalHash: sha256(canonical),113 semanticHash: simhash(ops.map((o) => o.key).join("\n")),114 publishedAt: null,115 extractionConfidence: 1,116 extra: { kind, version, operations: ops.length, deprecated: ops.filter((o) => o.deprecated).length, paths: Object.keys(spec.paths as object).length, servers: (spec.servers as { url: string }[] | undefined)?.map((s) => s.url) ?? (spec.host ? [String(spec.host)] : []) },117 };118 }119}120