TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { readFileSync } from "node:fs";2import { dirname, join } from "node:path";3import { fileURLToPath } from "node:url";4import { describe, expect, it } from "vitest";5import type { SensorEndpoint } from "@websensor/core";6import { expandUrl, JsonListConnector } from "./jsonlist";7import { parseFeed, RssConnector } from "./rss";8import { parseSitemap } from "./sitemap";9import { StatuspageConnector } from "./statuspage";10import { NormalizeError } from "./types";1112const here = dirname(fileURLToPath(import.meta.url));13const fx = (name: string): string => readFileSync(join(here, "..", "..", "..", "tests", "fixtures", name), "utf8");14const obs = (sensorId: string, body: string, contentType = "application/xml") => ({ sensorId, url: "https://example.com", fetchedAt: new Date("2026-09-08T12:00:00Z"), notModified: false, body: Buffer.from(body), meta: { status: 200, url: "https://example.com", finalUrl: "https://example.com", contentType, contentLength: body.length, etag: null, lastModified: null, durationMs: 10, redirects: 0, method: "GET" as const, headers: {} } });15const ep = (type: SensorEndpoint["type"], connector: string, config: Record<string, unknown> = {}, state: Record<string, unknown> | null = null): SensorEndpoint => ({ id: "t", sourceId: "acme", name: "t", url: "https://example.com", type, tier: "A", connector, config, state });1617describe("feed parsing", () => {18 it("parses RSS 2.0 with GUIDs, dates and HTML descriptions", () => {19 const f = parseFeed(fx("rss-before.xml"));20 expect(f.kind).toBe("rss");21 expect(f.items).toHaveLength(2);22 expect(f.items[0]).toMatchObject({ key: "post-2", title: "Second post", url: "https://acme.com/blog/second" });23 expect(f.items[0]!.summary).toBe("Body of second post with bold text.");24 expect(f.items[0]!.publishedAt).toBe("2026-09-07T10:00:00.000Z");25 });26 it("parses Atom and JSON Feed", () => {27 const a = parseFeed(fx("atom.xml"));28 expect(a.kind).toBe("atom");29 expect(a.items[0]).toMatchObject({ key: "tag:acme.com,2026:rel-2.1.0", title: "v2.1.0", url: "https://github.com/acme/tool/releases/tag/v2.1.0" });30 const j = parseFeed('{"version":"https://jsonfeed.org/version/1.1","title":"J","items":[{"id":"1","url":"https://a.com/1","title":"One","date_published":"2026-09-01T00:00:00Z"}]}');31 expect(j.kind).toBe("jsonfeed");32 expect(j.items[0]!.publishedAt).toBe("2026-09-01T00:00:00.000Z");33 });34 it("rejects HTML masquerading as a feed", async () => {35 const c = new RssConnector();36 await expect(c.normalize(ep("RSS", "rss"), obs("t", "<!doctype html><html><body>nope</body></html>", "text/html"))).rejects.toBeInstanceOf(NormalizeError);37 });38 it("normalizes to a keyed list and tracks seen keys across runs", async () => {39 const c = new RssConnector();40 const n1 = await c.normalize(ep("RSS", "rss"), obs("t", fx("rss-before.xml")));41 expect(n1.mode).toBe("list");42 expect(n1.items?.map((i) => i.key)).toEqual(["post-2", "post-1"]);43 const n2 = await c.normalize(ep("RSS", "rss", {}, n1.state ?? null), obs("t", fx("rss-after.xml")));44 expect(n2.canonicalHash).not.toBe(n1.canonicalHash);45 expect((n2.state?.seenKeys as string[]).sort()).toEqual(["post-1", "post-2", "post-3"]);46 expect(n2.publishedAt?.toISOString()).toBe("2026-09-08T09:30:00.000Z");47 });48});4950describe("sitemap parsing", () => {51 it("parses url sets, indexes and news metadata", () => {52 const s = parseSitemap(fx("sitemap.xml"));53 expect(s.kind).toBe("urlset");54 expect(s.entries).toHaveLength(3);55 expect(s.entries[0]).toMatchObject({ url: "https://acme.com/news/launch", lastmod: "2026-09-08" });56 expect(s.entries[0]!.title).toBe("Acme launches Widget 2");57 const idx = parseSitemap('<?xml version="1.0"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://acme.com/s1.xml</loc></sitemap></sitemapindex>');58 expect(idx.kind).toBe("sitemapindex");59 expect(idx.children).toEqual(["https://acme.com/s1.xml"]);60 expect(() => parseSitemap("<html></html>")).toThrow(NormalizeError);61 });62});6364describe("statuspage", () => {65 it("emits incidents, maintenances, degraded components and the overall indicator", async () => {66 const c = new StatuspageConnector();67 const n = await c.normalize(ep("STATUSPAGE", "statuspage"), obs("t", fx("statuspage.json"), "application/json"));68 const keys = n.items!.map((i) => i.key);69 expect(keys).toEqual(expect.arrayContaining(["incident:inc1", "maintenance:m1", "component:c2", "overall"]));70 expect(keys).not.toContain("component:c1"); // operational components are not tracked71 expect(n.items!.find((i) => i.key === "incident:inc1")).toMatchObject({ status: "investigating", impact: "major" });72 expect(n.extra).toMatchObject({ indicator: "major", activeIncidents: 1, degradedComponents: 1 });73 });74});7576describe("json list connector", () => {77 it("maps KEV-style records with templates and dot paths", async () => {78 const c = new JsonListConnector();79 const n = await c.normalize(ep("REST_API", "jsonlist", { itemsPath: "vulnerabilities", keyField: "cveID", titleTemplate: "{cveID} — {vendorProject} {product}", summaryField: "shortDescription", dateField: "dateAdded", urlTemplate: "https://kev.example/{key}", compareFields: ["knownRansomwareCampaignUse"] }), obs("t", fx("kev.json"), "application/json"));80 expect(n.items).toHaveLength(2);81 expect(n.items![0]).toMatchObject({ key: "CVE-2026-0001", title: "CVE-2026-0001 — Acme Router", url: "https://kev.example/CVE-2026-0001", knownRansomwareCampaignUse: "Known" });82 expect(n.publishedAt?.toISOString()).toBe("2026-09-08T00:00:00.000Z");83 await expect(c.normalize(ep("REST_API", "jsonlist", { itemsPath: "nope" }), obs("t", "{}", "application/json"))).rejects.toThrow(/not an array/);84 });85 it("expands time placeholders", () => {86 const u = expandUrl("https://api.example/cves?start={now-2h}&end={now}");87 expect(u).toMatch(/start=\d{4}-\d{2}-\d{2}T\d{2}%3A\d{2}%3A\d{2}\.000&end=/);88 });89});90