import YAML from "yaml"; import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core"; import { httpFetchWithRetry } from "./fetcher"; import { NormalizeError, type WebSensorConnector } from "./types"; /** * OpenAPI / Swagger connector. Fetches a public API description (JSON or YAML, OpenAPI 3.x or * Swagger 2.0) and normalizes it to a keyed list of operations (`GET /v1/charges`) with a stable * fingerprint of each operation's contract (parameters, request body, response codes, deprecation, * summary). Adding, removing or deprecating an endpoint, or changing its schema, becomes an * `api_change` list event; the `info` item tracks the declared API version. * * Specs can be large (Stripe ≈ 8 MB, GitHub ≈ 13 MB) — 64 MB limit, 60 s timeout. * Config: { maxOperations?: 5000, pathPrefix?: "/v1" } */ interface Op { key: string; [k: string]: unknown; method: string; path: string; title: string; summary: string; url: string; deprecated: boolean; fingerprint: string; publishedAt: null; } export function parseSpec(text: string, contentType: string | null): Record { const t = text.trimStart(); if (t.startsWith("{") || (contentType ?? "").includes("json")) { try { return JSON.parse(text) as Record; } catch { /* fall through to YAML */ } } try { const y = YAML.parse(text, { maxAliasCount: -1 }) as unknown; if (y && typeof y === "object") return y as Record; } catch (e) { throw new NormalizeError("bad_spec", `Not JSON nor YAML: ${(e as Error).message.slice(0, 120)}`); } throw new NormalizeError("bad_spec", "Document is not an object"); } const METHODS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"]; /** Stable JSON with sorted keys, descriptions/examples stripped (prose edits are not contract changes). */ function fingerprint(v: unknown): string { const strip = (x: unknown): unknown => { if (Array.isArray(x)) return x.map(strip); if (x && typeof x === "object") { const out: Record = {}; for (const k of Object.keys(x as object).sort()) { if (k === "description" || k === "example" || k === "examples" || k === "x-codeSamples" || k === "externalDocs" || k.startsWith("x-")) continue; out[k] = strip((x as Record)[k]); } return out; } return x; }; return sha256(JSON.stringify(strip(v))).slice(0, 16); } export function extractOperations(spec: Record, opts: { pathPrefix?: string; maxOperations?: number; baseUrl?: string } = {}): { ops: Op[]; info: Record; version: string; kind: string } { const paths = (spec.paths ?? {}) as Record>; const info = (spec.info ?? {}) as Record; const kind = spec.openapi ? `openapi ${String(spec.openapi)}` : spec.swagger ? `swagger ${String(spec.swagger)}` : "unknown"; const ops: Op[] = []; const max = opts.maxOperations ?? 5000; for (const p of Object.keys(paths).sort()) { if (opts.pathPrefix && !p.startsWith(opts.pathPrefix)) continue; const item = paths[p] ?? {}; const shared = { parameters: item.parameters }; for (const m of METHODS) { const op = item[m] as Record | undefined; if (!op) continue; const method = m.toUpperCase(); const responses = Object.keys((op.responses ?? {}) as object).sort(); const fp = fingerprint({ parameters: op.parameters, shared, requestBody: op.requestBody, responses: op.responses, security: op.security, deprecated: op.deprecated, operationId: op.operationId, summary: op.summary }); ops.push({ key: `${method} ${p}`, method, path: p, title: `${method} ${p}${op.deprecated ? " (deprecated)" : ""}`, summary: String(op.summary ?? op.operationId ?? "").slice(0, 300), url: opts.baseUrl ?? "", deprecated: Boolean(op.deprecated), tags: (op.tags as string[] | undefined) ?? [], responses: responses.join(","), fingerprint: fp, publishedAt: null }); if (ops.length >= max) return { ops, info, version: String(info.version ?? ""), kind }; } } return { ops, info, version: String(info.version ?? ""), kind }; } export class OpenApiConnector implements WebSensorConnector { mode = "list" as const; metadata(): ConnectorMetadata { return { key: "openapi", name: "OpenAPI / Swagger", sensorTypes: ["JSON", "FILE"], description: "API description → operations list with contract fingerprints (added/removed/deprecated/changed endpoints, version)", version: "1.0.0" }; } async fetch(endpoint: SensorEndpoint): Promise { return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/json, application/yaml, text/yaml, text/plain;q=0.8, */*;q=0.5", timeoutMs: 60_000, maxBytes: 64 * 1024 * 1024 }); } async normalize(endpoint: SensorEndpoint, obs: Observation): Promise { if (!obs.body) throw new NormalizeError("no_body", "Observation has no body"); const text = obs.body.toString("utf8"); const spec = parseSpec(text, obs.meta.contentType); if (!spec.paths || typeof spec.paths !== "object") throw new NormalizeError("bad_spec", "No `paths` object — not an OpenAPI/Swagger document"); const cfg = endpoint.config as { pathPrefix?: string; maxOperations?: number; docsUrl?: string }; const { ops, info, version, kind } = extractOperations(spec, { pathPrefix: cfg.pathPrefix, maxOperations: cfg.maxOperations, baseUrl: cfg.docsUrl ?? endpoint.url }); const items: { key: string; [k: string]: unknown }[] = [{ key: "info", kind: "info", title: `${String(info.title ?? "API")} — version ${version || "?"}`, version, fingerprint: version, summary: String(info.description ?? "").slice(0, 300), url: cfg.docsUrl ?? endpoint.url, deprecated: false, publishedAt: null }, ...ops]; const canonical = items.map((i) => `${i.key}\t${String(i.fingerprint)}\t${String(i.deprecated)}`).join("\n"); return { mode: "list", items, compareFields: ["fingerprint", "deprecated"], title: String(info.title ?? endpoint.name), rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(ops.map((o) => o.key).join("\n")), publishedAt: null, extractionConfidence: 1, extra: { kind, version, operations: ops.length, deprecated: ops.filter((o) => o.deprecated).length, paths: Object.keys(spec.paths as object).length, servers: (spec.servers as { url: string }[] | undefined)?.map((s) => s.url) ?? (spec.host ? [String(spec.host)] : []) }, }; } }