TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { describe, expect, it } from "vitest";2import { canonicalizeHtml, canonicalizeText, stripTrackingParams } from "./canonical";3import { diffJson, diffList, diffText, summarizeDiff } from "./diff";4import { hammingHex, jaccard, shingles, simhash } from "./hash";5import { describeChange, evaluateChange } from "./heuristics";6import { computeConfidence, computeImportance, sourceImportanceFromTier } from "./scoring";7import { nextIntervalSeconds } from "./schedule";8import { assertUrlAllowed, isBlockedHostname, isBlockedIP } from "./ssrf";9import { slugify } from "./ids";1011const page = (price: string, year: string, extra = "") => `<!doctype html><html><head><title>API Pricing · Acme</title><meta name="description" content="Prices"></head>12<body><nav><a href="/">Home</a><a href="/blog">Blog</a></nav><main><h1>API Pricing</h1><p>Input: ${price} / million tokens</p><p>Output: $30 / million tokens</p>${extra}13<script>window.__t=${Date.now()}</script></main><footer>© ${year} Acme · <span>12,345 visitors online</span></footer></body></html>`;1415describe("canonicalization", () => {16 it("ignores render noise (scripts, nav, copyright year, counters, tracking params)", () => {17 const a = canonicalizeHtml(page("$10", "2025"), "https://acme.com/pricing");18 const b = canonicalizeHtml(page("$10", "2026"), "https://acme.com/pricing");19 expect(a.canonicalHash).toBe(b.canonicalHash);20 expect(a.rawHash).not.toBe(b.rawHash);21 expect(a.title).toBe("API Pricing · Acme");22 expect(a.headings).toEqual(["API Pricing"]);23 expect(stripTrackingParams("https://x.com/a?utm_source=t&id=3&fbclid=9#frag")).toBe("https://x.com/a?id=3");24 });25 it("changes the canonical hash when content changes", () => {26 const a = canonicalizeHtml(page("$10", "2026"));27 const b = canonicalizeHtml(page("$8", "2026"));28 expect(a.canonicalHash).not.toBe(b.canonicalHash);29 expect(hammingHex(a.semanticHash, b.semanticHash)).toBeLessThan(12);30 });31 it("normalizes plain text", () => {32 const t = canonicalizeText("a b\r\n\r\n c \n");33 expect(t.text).toBe("a b\nc");34 });35});3637describe("diff engines", () => {38 it("pairs modified lines", () => {39 const d = diffText("Input: $10 / million tokens\nOutput: $30\nUnchanged", "Input: $8 / million tokens\nOutput: $30\nUnchanged");40 expect(d.modified).toEqual([{ before: "Input: $10 / million tokens", after: "Input: $8 / million tokens" }]);41 expect(d.added).toEqual([]);42 expect(d.stats.unchangedRatio).toBeGreaterThanOrEqual(0.5);43 });44 it("diffs JSON structurally", () => {45 const d = diffJson({ price: { input: 10, output: 30 }, models: ["a"] }, { price: { input: 8, output: 30 }, models: ["a", "b"], region: "eu" });46 expect(d.changes).toEqual(expect.arrayContaining([{ path: "price.input", op: "replace", before: 10, after: 8 }, { path: "region", op: "add", after: "eu" }]));47 });48 it("diffs keyed lists", () => {49 const d = diffList([{ key: "1", title: "A", status: "open" }, { key: "2", title: "B" }], [{ key: "1", title: "A", status: "resolved" }, { key: "3", title: "C" }], ["status"]);50 expect(d.added.map((i) => i.key)).toEqual(["3"]);51 expect(d.removed.map((i) => i.key)).toEqual(["2"]);52 expect(d.modified[0]).toMatchObject({ key: "1", fields: ["status"] });53 expect(summarizeDiff(d)).toMatchObject({ kind: "list", counts: { added: 1, removed: 1, modified: 1 } });54 });55});5657describe("heuristics", () => {58 it("scores a copyright bump as noise", () => {59 const d = diffText("© 2025 Acme\nAll rights reserved", "© 2026 Acme\nAll rights reserved");60 const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com", sourceCategories: [] });61 expect(h.signal).toBeLessThan(0.1);62 expect(h.eventType).toBe("unknown");63 });64 it("detects a pricing change with extracted facts", () => {65 const d = diffText("Input: $10 / million tokens", "Input: $8 / million tokens");66 const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com/pricing", sourceCategories: ["ai"] });67 expect(h.eventType).toBe("pricing_change");68 expect(h.signal).toBeGreaterThan(0.5);69 expect(h.facts[0]).toMatchObject({ kind: "price", before: "$10 / million tokens", after: "$8 / million tokens" });70 const desc = describeChange(h, d, { sourceName: "Acme", url: "https://acme.com/pricing", sensorName: "pricing" });71 expect(desc.title).toContain("$10 / million tokens → $8 / million tokens");72 });73 it("classifies new status incidents and CVEs", () => {74 const inc = diffList([], [{ key: "incident:1", title: "Elevated error rates — investigating", summary: "We are investigating elevated error rates on the API." }]);75 expect(evaluateChange(inc, { sensorType: "STATUSPAGE", url: "https://status.acme.com/api/v2/summary.json", sourceCategories: ["ai"] }).eventType).toBe("incident");76 const cve = diffList([], [{ key: "CVE-2026-1234", title: "CVE-2026-1234 — Acme Router: Remote Code Execution Vulnerability", summary: "Acme Router contains an RCE vulnerability." }]);77 expect(evaluateChange(cve, { sensorType: "REST_API", url: "https://kev.example", sourceCategories: ["cyber"] }).eventType).toBe("vulnerability");78 });79});8081describe("scoring", () => {82 it("weights components and stays in range", () => {83 const r = computeImportance({ eventType: "pricing_change", sourceImportance: 92, entityImportance: 90, novelty: 90, magnitude: 60, confirmations: 1 });84 expect(r.score).toBeGreaterThan(75);85 expect(r.score).toBeLessThanOrEqual(100);86 expect(Object.keys(r.components)).toHaveLength(8);87 const low = computeImportance({ eventType: "content_change", sourceImportance: 35, entityImportance: 30, novelty: 10, magnitude: 5, confirmations: 0 });88 expect(low.score).toBeLessThan(35);89 expect(sourceImportanceFromTier("S")).toBe(92);90 expect(computeConfidence({ sourceAuthenticity: 1, extraction: 1, diffClarity: 1, structured: true, confirmations: 2, llmAgreement: 1 })).toBe(100);91 });92});9394describe("adaptive schedule", () => {95 it("tightens after recent changes and backs off after errors", () => {96 const base = { tier: "B" as const, changes7d: 0, events7d: 0, consecutiveErrors: 0, lastWas304: false };97 const quiet = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 30 * 86400e3) });98 const burst = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 5 * 60e3), changes7d: 30 });99 const failing = nextIntervalSeconds({ ...base, lastChangeAt: null, consecutiveErrors: 4 });100 expect(burst).toBeLessThan(quiet);101 expect(burst).toBeGreaterThanOrEqual(300);102 expect(failing).toBeGreaterThan(quiet);103 });104});105106describe("ssrf policy", () => {107 it("blocks private ranges, metadata and cluster hosts", async () => {108 expect(isBlockedIP("10.1.2.3")).toBe(true);109 expect(isBlockedIP("169.254.169.254")).toBe(true);110 expect(isBlockedIP("100.64.0.1")).toBe(true);111 expect(isBlockedIP("::ffff:192.168.1.1")).toBe(true);112 expect(isBlockedIP("8.8.8.8")).toBe(false);113 // NAT64: the embedded IPv4 decides (64:ff9b::8c52:7004 = 140.82.112.4 public; 64:ff9b::a00:1 = 10.0.0.1 private)114 expect(isBlockedIP("64:ff9b::8c52:7004")).toBe(false);115 expect(isBlockedIP("64:ff9b::a00:1")).toBe(true);116 expect(isBlockedIP("64:ff9b::a9fe:a9fe")).toBe(true);117 expect(isBlockedHostname("m3u96a.maclustr.io")).toBe(true);118 expect(isBlockedHostname("localhost")).toBe(true);119 await expect(assertUrlAllowed("http://127.0.0.1:8080/")).rejects.toThrow(/not allowed/);120 await expect(assertUrlAllowed("ftp://example.com")).rejects.toThrow(/Scheme/);121 await expect(assertUrlAllowed("https://user:pw@example.com")).rejects.toThrow(/Credentials/);122 });123});124125describe("hashing", () => {126 it("shingles + jaccard measure similarity", () => {127 const a = shingles("OpenAI releases new model GPT-5 with lower pricing");128 const b = shingles("OpenAI releases new model GPT-5 with lower prices today");129 const c = shingles("FDA approves a new drug for diabetes");130 expect(jaccard(a, b)).toBeGreaterThan(0.3);131 expect(jaccard(a, c)).toBe(0);132 expect(simhash("x")).toHaveLength(16);133 expect(slugify("Anthropic: Claude Opus 5 — pricing changed!")).toBe("anthropic-claude-opus-5-pricing-changed");134 });135});136137describe("connector-class heuristic priors", () => {138 it("classifies TLS/DNS json changes and EDGAR/package/openapi list items", () => {139 const jsonDiff = { kind: "json" as const, changes: [{ path: "issuer.O", before: "Let's Encrypt", after: "DigiCert Inc", op: "changed" as const }], stats: { added: 0, removed: 0, modified: 1, unchangedRatio: 0.9 } };140 expect(evaluateChange(jsonDiff as never, { sensorType: "TLS", url: "tls://www.example.com", sourceCategories: ["technology"] }).eventType).toBe("certificate_change");141 expect(evaluateChange(jsonDiff as never, { sensorType: "DNS", url: "dns://example.com", sourceCategories: ["technology"] }).eventType).toBe("dns_change");142 const filing = { kind: "list" as const, added: [{ key: "0001-26-1", form: "8-K", items: "2.02,9.01", title: "Acme: 8-K — Current report (2.02)", summary: "Results of operations", url: "https://www.sec.gov/x" }], removed: [], modified: [], stats: { added: 1, removed: 0, modified: 0, unchangedRatio: 0.98 } };143 expect(evaluateChange(filing as never, { sensorType: "REST_API", url: "https://data.sec.gov/submissions/CIK0000000001.json", sourceCategories: ["technology"] }).eventType).toBe("earnings");144 const pkg = { kind: "list" as const, added: [{ key: "19.3.0", version: "19.3.0", prerelease: false, title: "react 19.3.0", url: "https://www.npmjs.com/package/react/v/19.3.0" }], removed: [], modified: [{ before: { key: "latest", version: "19.2.0" }, after: { key: "latest", version: "19.3.0", prerelease: false, title: "react latest → 19.3.0" } }], stats: { added: 1, removed: 0, modified: 1, unchangedRatio: 0.95 } };145 expect(evaluateChange(pkg as never, { sensorType: "REST_API", url: "https://www.npmjs.com/package/react", sourceCategories: ["open-source"] }).eventType).toBe("software_release");146 const api = { kind: "list" as const, added: [{ key: "POST /v1/agents", fingerprint: "abc", deprecated: false, title: "POST /v1/agents", url: "https://x" }], removed: [], modified: [], stats: { added: 1, removed: 0, modified: 0, unchangedRatio: 0.99 } };147 expect(evaluateChange(api as never, { sensorType: "JSON", url: "https://raw.githubusercontent.com/x/openapi.json", sourceCategories: ["ai"] }).eventType).toBe("api_change");148 const robots = { kind: "text" as const, added: ["User-agent: GPTBot", "Disallow: /"], removed: [], modified: [], stats: { added: 2, removed: 0, modified: 0, unchangedRatio: 0.8 } };149 const r = evaluateChange(robots as never, { sensorType: "FILE", url: "https://www.example.com/robots.txt", sourceCategories: ["media"] });150 expect(r.eventType).toBe("crawler_policy_change");151 expect(r.reasons).toContain("AI crawler directive changed");152 });153});154