1import { describe, expect, it } from "vitest";2import { canonicalizeHtml, canonicalizeText, stripTrackingParams } from "./canonical";3import { diffJson, diffList, diffText, summarizeDiff } from "./diff";4import { hammingHex, jaccard, shingles, simhash } from "./hash";5import { describeChange, evaluateChange } from "./heuristics";6import { computeConfidence, computeImportance, sourceImportanceFromTier } from "./scoring";7import { nextIntervalSeconds } from "./schedule";8import { assertUrlAllowed, isBlockedHostname, isBlockedIP } from "./ssrf";9import { slugify } from "./ids";1011const page = (price: string, year: string, extra = "") => `<!doctype html><html><head><title>API Pricing · Acme</title><meta name="description" content="Prices"></head>12<body><nav><a href="/">Home</a><a href="/blog">Blog</a></nav><main><h1>API Pricing</h1><p>Input: ${price} / million tokens</p><p>Output: $30 / million tokens</p>${extra}13<script>window.__t=${Date.now()}</script></main><footer>© ${year} Acme · <span>12,345 visitors online</span></footer></body></html>`;1415describe("canonicalization", () => {16 it("ignores render noise (scripts, nav, copyright year, counters, tracking params)", () => {17 const a = canonicalizeHtml(page("$10", "2025"), "https://acme.com/pricing");18 const b = canonicalizeHtml(page("$10", "2026"), "https://acme.com/pricing");19 expect(a.canonicalHash).toBe(b.canonicalHash);20 expect(a.rawHash).not.toBe(b.rawHash);21 expect(a.title).toBe("API Pricing · Acme");22 expect(a.headings).toEqual(["API Pricing"]);23 expect(stripTrackingParams("https://x.com/a?utm_source=t&id=3&fbclid=9#frag")).toBe("https://x.com/a?id=3");24 });25 it("changes the canonical hash when content changes", () => {26 const a = canonicalizeHtml(page("$10", "2026"));27 const b = canonicalizeHtml(page("$8", "2026"));28 expect(a.canonicalHash).not.toBe(b.canonicalHash);29 expect(hammingHex(a.semanticHash, b.semanticHash)).toBeLessThan(12);30 });31 it("normalizes plain text", () => {32 const t = canonicalizeText("a b\r\n\r\n c \n");33 expect(t.text).toBe("a b\nc");34 });35});3637describe("diff engines", () => {38 it("pairs modified lines", () => {39 const d = diffText("Input: $10 / million tokens\nOutput: $30\nUnchanged", "Input: $8 / million tokens\nOutput: $30\nUnchanged");40 expect(d.modified).toEqual([{ before: "Input: $10 / million tokens", after: "Input: $8 / million tokens" }]);41 expect(d.added).toEqual([]);42 expect(d.stats.unchangedRatio).toBeGreaterThanOrEqual(0.5);43 });44 it("diffs JSON structurally", () => {45 const d = diffJson({ price: { input: 10, output: 30 }, models: ["a"] }, { price: { input: 8, output: 30 }, models: ["a", "b"], region: "eu" });46 expect(d.changes).toEqual(expect.arrayContaining([{ path: "price.input", op: "replace", before: 10, after: 8 }, { path: "region", op: "add", after: "eu" }]));47 });48 it("diffs keyed lists", () => {49 const d = diffList([{ key: "1", title: "A", status: "open" }, { key: "2", title: "B" }], [{ key: "1", title: "A", status: "resolved" }, { key: "3", title: "C" }], ["status"]);50 expect(d.added.map((i) => i.key)).toEqual(["3"]);51 expect(d.removed.map((i) => i.key)).toEqual(["2"]);52 expect(d.modified[0]).toMatchObject({ key: "1", fields: ["status"] });53 expect(summarizeDiff(d)).toMatchObject({ kind: "list", counts: { added: 1, removed: 1, modified: 1 } });54 });55});5657describe("heuristics", () => {58 it("scores a copyright bump as noise", () => {59 const d = diffText("© 2025 Acme\nAll rights reserved", "© 2026 Acme\nAll rights reserved");60 const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com", sourceCategories: [] });61 expect(h.signal).toBeLessThan(0.1);62 expect(h.eventType).toBe("unknown");63 });64 it("detects a pricing change with extracted facts", () => {65 const d = diffText("Input: $10 / million tokens", "Input: $8 / million tokens");66 const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com/pricing", sourceCategories: ["ai"] });67 expect(h.eventType).toBe("pricing_change");68 expect(h.signal).toBeGreaterThan(0.5);69 expect(h.facts[0]).toMatchObject({ kind: "price", before: "$10 / million tokens", after: "$8 / million tokens" });70 const desc = describeChange(h, d, { sourceName: "Acme", url: "https://acme.com/pricing", sensorName: "pricing" });71 expect(desc.title).toContain("$10 / million tokens → $8 / million tokens");72 });73 it("classifies new status incidents and CVEs", () => {74 const inc = diffList([], [{ key: "incident:1", title: "Elevated error rates — investigating", summary: "We are investigating elevated error rates on the API." }]);75 expect(evaluateChange(inc, { sensorType: "STATUSPAGE", url: "https://status.acme.com/api/v2/summary.json", sourceCategories: ["ai"] }).eventType).toBe("incident");76 const cve = diffList([], [{ key: "CVE-2026-1234", title: "CVE-2026-1234 — Acme Router: Remote Code Execution Vulnerability", summary: "Acme Router contains an RCE vulnerability." }]);77 expect(evaluateChange(cve, { sensorType: "REST_API", url: "https://kev.example", sourceCategories: ["cyber"] }).eventType).toBe("vulnerability");78 });79});8081describe("scoring", () => {82 it("weights components and stays in range", () => {83 const r = computeImportance({ eventType: "pricing_change", sourceImportance: 92, entityImportance: 90, novelty: 90, magnitude: 60, confirmations: 1 });84 expect(r.score).toBeGreaterThan(75);85 expect(r.score).toBeLessThanOrEqual(100);86 expect(Object.keys(r.components)).toHaveLength(8);87 const low = computeImportance({ eventType: "content_change", sourceImportance: 35, entityImportance: 30, novelty: 10, magnitude: 5, confirmations: 0 });88 expect(low.score).toBeLessThan(35);89 expect(sourceImportanceFromTier("S")).toBe(92);90 expect(computeConfidence({ sourceAuthenticity: 1, extraction: 1, diffClarity: 1, structured: true, confirmations: 2, llmAgreement: 1 })).toBe(100);91 });92});9394describe("adaptive schedule", () => {95 it("tightens after recent changes and backs off after errors", () => {96 const base = { tier: "B" as const, changes7d: 0, events7d: 0, consecutiveErrors: 0, lastWas304: false };97 const quiet = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 30 * 86400e3) });98 const burst = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 5 * 60e3), changes7d: 30 });99 const failing = nextIntervalSeconds({ ...base, lastChangeAt: null, consecutiveErrors: 4 });100 expect(burst).toBeLessThan(quiet);101 expect(burst).toBeGreaterThanOrEqual(300);102 expect(failing).toBeGreaterThan(quiet);103 });104});105106describe("ssrf policy", () => {107 it("blocks private ranges, metadata and cluster hosts", async () => {108 expect(isBlockedIP("10.1.2.3")).toBe(true);109 expect(isBlockedIP("169.254.169.254")).toBe(true);110 expect(isBlockedIP("100.64.0.1")).toBe(true);111 expect(isBlockedIP("::ffff:192.168.1.1")).toBe(true);112 expect(isBlockedIP("8.8.8.8")).toBe(false);113 expect(isBlockedHostname("m3u96a.maclustr.io")).toBe(true);114 expect(isBlockedHostname("localhost")).toBe(true);115 await expect(assertUrlAllowed("http://127.0.0.1:8080/")).rejects.toThrow(/not allowed/);116 await expect(assertUrlAllowed("ftp://example.com")).rejects.toThrow(/Scheme/);117 await expect(assertUrlAllowed("https://user:pw@example.com")).rejects.toThrow(/Credentials/);118 });119});120121describe("hashing", () => {122 it("shingles + jaccard measure similarity", () => {123 const a = shingles("OpenAI releases new model GPT-5 with lower pricing");124 const b = shingles("OpenAI releases new model GPT-5 with lower prices today");125 const c = shingles("FDA approves a new drug for diabetes");126 expect(jaccard(a, b)).toBeGreaterThan(0.3);127 expect(jaccard(a, c)).toBe(0);128 expect(simhash("x")).toHaveLength(16);129 expect(slugify("Anthropic: Claude Opus 5 — pricing changed!")).toBe("anthropic-claude-opus-5-pricing-changed");130 });131});132