TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { describe, expect, it } from "vitest";2import type { SensorEndpoint } from "@websensor/core";3import { EdgarConnector } from "./edgar";4import { PackageConnector, registryApiUrl, semverCompare } from "./package";5import { detectFlavor, normalizeStatusJson, StatusJsonConnector } from "./statusjson";6import { extractOperations, parseSpec } from "./openapi";7import { CsvConnector, detectDelimiter, parseDelimited } from "./csv";8import { summarizeRdap } from "./rdap";9import { hostOf } from "./dns";10import { getConnector, listConnectors } from "./index";11import { NormalizeError } from "./types";1213const obs = (sensorId: string, body: string, contentType = "application/json") => ({ sensorId, url: "https://example.com", fetchedAt: new Date("2026-09-08T12:00:00Z"), notModified: false, body: Buffer.from(body), meta: { status: 200, url: "https://example.com", finalUrl: "https://example.com", contentType, contentLength: body.length, etag: null, lastModified: null, durationMs: 10, redirects: 0, method: "GET" as const, headers: {} } });14const ep = (type: SensorEndpoint["type"], connector: string, config: Record<string, unknown> = {}): SensorEndpoint => ({ id: "t", sourceId: "acme", name: "t", url: "https://example.com", type, tier: "A", connector, config, state: null });1516describe("connector registry", () => {17 it("exposes the 16 connector families", () => {18 const keys = listConnectors().map((c) => c.metadata().key).sort();19 expect(keys).toEqual(["csv", "dns", "edgar", "github", "headers", "http", "jsonlist", "openapi", "package", "pdf", "rdap", "rss", "sitemap", "statusjson", "statuspage", "tls"]);20 expect(() => getConnector("nope")).toThrow(/Unknown connector/);21 });22});2324describe("package connector", () => {25 it("derives registry API urls", () => {26 expect(registryApiUrl("npm", "@types/node")).toBe("https://registry.npmjs.org/@types%2Fnode");27 expect(registryApiUrl("packagist", "laravel/framework")).toBe("https://repo.packagist.org/p2/laravel/framework.json");28 expect(registryApiUrl("dockerhub", "nginx")).toContain("/library/nginx/tags");29 expect(registryApiUrl("goproxy", "github.com/Gin-Gonic/gin")).toBe("https://proxy.golang.org/github.com/gin-gonic/gin/@v/list");30 });31 it("normalizes npm metadata to a version list with a latest item", async () => {32 const c = new PackageConnector();33 const body = JSON.stringify({ name: "react", description: "UI lib", "dist-tags": { latest: "19.2.0", next: "19.3.0-canary" }, versions: { "19.1.0": {}, "19.2.0": {}, "19.3.0-canary": {} }, time: { created: "2011-01-01T00:00:00Z", "19.1.0": "2026-01-01T00:00:00Z", "19.2.0": "2026-06-01T00:00:00Z", "19.3.0-canary": "2026-09-01T00:00:00Z" } });34 const n = await c.normalize(ep("REST_API", "package", { registry: "npm", name: "react" }), obs("t", body));35 expect(n.mode).toBe("list");36 expect(n.items?.[0]).toMatchObject({ key: "latest", version: "19.2.0" });37 expect(n.items?.map((i) => i.key)).toContain("19.3.0-canary");38 expect(n.items?.find((i) => i.key === "19.3.0-canary")?.prerelease).toBe(true);39 expect(n.compareFields).toEqual(["version"]);40 // A new latest changes the canonical hash even when the version list is unchanged.41 const body2 = body.replace('"latest":"19.2.0"', '"latest":"19.3.0-canary"');42 const n2 = await c.normalize(ep("REST_API", "package", { registry: "npm", name: "react" }), obs("t", body2));43 expect(n2.canonicalHash).not.toBe(n.canonicalHash);44 });45 it("parses the Go proxy text list and sorts by semver", async () => {46 const c = new PackageConnector();47 const n = await c.normalize(ep("REST_API", "package", { registry: "goproxy", name: "github.com/x/y" }), obs("t", "v1.9.0\nv1.10.0\nv1.10.1-rc1\n", "text/plain"));48 expect(n.items?.[1]?.key).toBe("v1.10.1-rc1");49 expect(n.items?.[0]).toMatchObject({ key: "latest", version: "v1.10.0" });50 expect(semverCompare("1.10.0", "1.9.0")).toBeGreaterThan(0);51 });52 it("rejects missing config", async () => {53 const c = new PackageConnector();54 await expect(c.normalize(ep("REST_API", "package"), obs("t", "{}"))).rejects.toBeInstanceOf(NormalizeError);55 });56});5758describe("edgar connector", () => {59 it("rebuilds filings from the columnar submissions JSON and decodes 8-K items", async () => {60 const c = new EdgarConnector();61 const body = JSON.stringify({ cik: "320193", name: "Apple Inc.", tickers: ["AAPL"], filings: { recent: { accessionNumber: ["0000320193-26-000001", "0000320193-26-000002", "0000320193-26-000003"], filingDate: ["2026-09-01", "2026-08-30", "2026-08-29"], reportDate: ["2026-08-31", "", ""], acceptanceDateTime: ["2026-09-01T20:30:00.000Z", "2026-08-30T21:00:00.000Z", "2026-08-29T10:00:00.000Z"], form: ["8-K", "4", "10-Q"], items: ["2.02,9.01", "", ""], primaryDocument: ["a8k.htm", "form4.xml", "a10q.htm"], primaryDocDescription: ["8-K", "FORM 4", "10-Q"], size: [1, 2, 3], isXBRL: [1, 0, 1] } } });62 const n = await c.normalize(ep("REST_API", "edgar"), obs("t", body));63 expect(n.items).toHaveLength(2); // form 4 excluded by default64 expect(n.items?.[0]).toMatchObject({ key: "0000320193-26-000001", form: "8-K", url: "https://www.sec.gov/Archives/edgar/data/320193/000032019326000001/a8k.htm" });65 expect(String(n.items?.[0]?.summary)).toContain("Results of operations");66 expect(n.title).toContain("AAPL");67 const only10q = await c.normalize(ep("REST_API", "edgar", { forms: ["10-Q"] }), obs("t", body));68 expect(only10q.items?.map((i) => i.form)).toEqual(["10-Q"]);69 });70});7172describe("statusjson connector", () => {73 it("detects and normalizes Instatus, incident.io and Status.io shapes", async () => {74 const instatus = { page: { name: "Acme", url: "https://status.acme.com", status: "HASISSUES" }, activeIncidents: [{ id: "i1", name: "API errors", status: "INVESTIGATING", impact: "MAJOROUTAGE", started: "2026-09-08T10:00:00Z", url: "https://status.acme.com/i1" }] };75 expect(detectFlavor(instatus)).toBe("instatus");76 const a = normalizeStatusJson(instatus, "instatus");77 expect(a.items.map((i) => i.key)).toEqual(["incident:i1", "overall"]);78 expect(a.items[0]).toMatchObject({ status: "investigating", impact: "majoroutage" });79 const incidentio = { page_title: "Acme", page_url: "https://status.acme.com", ongoing_incidents: [], in_progress_maintenances: [], scheduled_maintenances: [{ id: "m1", name: "DB upgrade", status: "scheduled", starts_at: "2026-09-10T02:00:00Z" }] };80 expect(detectFlavor(incidentio)).toBe("incidentio");81 const b = normalizeStatusJson(incidentio, "incidentio");82 expect(b.items.map((i) => i.kind)).toEqual(["maintenance", "overall"]);83 expect(b.indicator).toBe("operational");84 const statusio = { result: { status_overall: { status: "Degraded Performance" }, status: [{ name: "API", containers: [{ id: "c1", name: "EU", status: "Degraded Performance", updated: "2026-09-08T09:00:00Z" }] }], incidents: [{ _id: "x1", name: "Latency", current_status: "Monitoring", current_impact: "Degraded", datetime_open: "2026-09-08T08:00:00Z", messages: [{ details: "Fix deployed", datetime: "2026-09-08T09:30:00Z" }] }], maintenance: { active: [], upcoming: [] } } };85 expect(detectFlavor(statusio)).toBe("statusio");86 const c = normalizeStatusJson(statusio, "statusio");87 expect(c.items.map((i) => i.key)).toEqual(["incident:x1", "component:c1", "overall"]);88 expect(c.items[0]?.summary).toBe("Fix deployed");89 const conn = new StatusJsonConnector();90 await expect(conn.normalize(ep("STATUSPAGE", "statusjson"), obs("t", '{"foo":1}'))).rejects.toThrow(/Unrecognized/);91 const n = await conn.normalize(ep("STATUSPAGE", "statusjson"), obs("t", JSON.stringify(instatus)));92 expect(n.extra?.activeIncidents).toBe(1);93 });94});9596describe("openapi connector", () => {97 it("extracts operations with contract fingerprints and ignores prose", () => {98 const spec = parseSpec('openapi: 3.1.0\ninfo: { title: Acme API, version: "2026-09-01" }\npaths:\n /v1/charges:\n get: { summary: List charges, responses: { "200": { description: ok } } }\n post: { summary: Create, deprecated: true, requestBody: { content: { application/json: { schema: { type: object } } } }, responses: { "200": {} } }\n', "text/yaml");99 const { ops, version, kind } = extractOperations(spec);100 expect(kind).toBe("openapi 3.1.0");101 expect(version).toBe("2026-09-01");102 expect(ops.map((o) => o.key)).toEqual(["GET /v1/charges", "POST /v1/charges"]);103 expect(ops[1]?.deprecated).toBe(true);104 const spec2 = parseSpec(JSON.stringify({ openapi: "3.0.0", info: { title: "A", version: "1" }, paths: { "/v1/charges": { get: { summary: "List charges", description: "edited prose", responses: { "200": { description: "ok" } } } } } }), "application/json");105 expect(extractOperations(spec2).ops[0]?.fingerprint).toBe(ops[0]?.fingerprint);106 const spec3 = parseSpec(JSON.stringify({ openapi: "3.0.0", info: { title: "A", version: "1" }, paths: { "/v1/charges": { get: { summary: "List charges", parameters: [{ name: "limit", in: "query" }], responses: { "200": { description: "ok" } } } } } }), "application/json");107 expect(extractOperations(spec3).ops[0]?.fingerprint).not.toBe(ops[0]?.fingerprint);108 expect(() => parseSpec("{ unclosed: [", null)).toThrow(NormalizeError);109 expect(() => parseSpec("just a string", null)).toThrow(/not an object/);110 });111});112113describe("csv connector", () => {114 it("parses quoted fields, detects delimiters and keys rows", async () => {115 expect(detectDelimiter("a\tb\n1\t2")).toBe("\t");116 expect(parseDelimited('id,name\n1,"Smith, John"\n2,"He said ""hi"""\n')).toEqual([["id", "name"], ["1", "Smith, John"], ["2", 'He said "hi"']]);117 const c = new CsvConnector();118 const body = "# comment\nobservation_date,FEDFUNDS\n2026-06-01,3.63\n2026-07-01,3.63\n2026-08-01,3.88\n";119 const n = await c.normalize(ep("FILE", "csv", { commentPrefix: "#", keyColumn: "observation_date", compareColumns: ["FEDFUNDS"], dateColumn: "observation_date", tail: true, maxRows: 2 }), obs("t", body, "text/csv"));120 expect(n.mode).toBe("list");121 expect(n.items?.map((i) => i.key)).toEqual(["2026-07-01", "2026-08-01"]);122 expect(n.items?.[1]?.FEDFUNDS).toBe("3.88");123 expect(n.publishedAt?.toISOString()).toBe("2026-08-01T00:00:00.000Z");124 await expect(c.normalize(ep("FILE", "csv", { keyColumn: "nope" }), obs("t", body, "text/csv"))).rejects.toThrow(/keyColumn/);125 await expect(c.normalize(ep("FILE", "csv"), obs("t", "<!doctype html><html></html>", "text/html"))).rejects.toThrow(/HTML/);126 });127});128129describe("rdap + dns helpers", () => {130 it("summarizes an RDAP domain object without the volatile fields", () => {131 const s = summarizeRdap({ objectClassName: "domain", ldhName: "EXAMPLE.COM", handle: "123", status: ["client transfer prohibited", "active"], nameservers: [{ ldhName: "NS2.EXAMPLE.NET." }, { ldhName: "ns1.example.net" }], secureDNS: { delegationSigned: true }, events: [{ eventAction: "registration", eventDate: "2001-10-02T18:10:32Z" }, { eventAction: "expiration", eventDate: "2033-10-02T18:10:32Z" }, { eventAction: "last update of RDAP database", eventDate: "2026-09-08T18:10:49Z" }], entities: [{ roles: ["registrar"], handle: "292", vcardArray: ["vcard", [["fn", {}, "text", "MarkMonitor Inc."]]], publicIds: [{ type: "IANA Registrar ID", identifier: "292" }] }] });132 expect(s).toMatchObject({ domain: "example.com", registrar: "MarkMonitor Inc.", registrarIanaId: "292", nameservers: ["ns1.example.net", "ns2.example.net"], dnssec: true, status: ["active", "client transfer prohibited"], expiration: "2033-10-02T18:10:32.000Z" });133 expect(JSON.stringify(s)).not.toContain("2026-09-08");134 });135 it("extracts hosts from dns:// and tls:// urls", () => {136 expect(hostOf("dns://Example.com.")).toBe("example.com");137 expect(hostOf("tls://www.anthropic.com:443")).toBe("www.anthropic.com");138 expect(hostOf("anthropic.com")).toBe("anthropic.com");139 });140});141