import { describe, expect, it } from "vitest";
import { canonicalizeHtml, canonicalizeText, stripTrackingParams } from "./canonical";
import { diffJson, diffList, diffText, summarizeDiff } from "./diff";
import { hammingHex, jaccard, shingles, simhash } from "./hash";
import { describeChange, evaluateChange } from "./heuristics";
import { computeConfidence, computeImportance, sourceImportanceFromTier } from "./scoring";
import { nextIntervalSeconds } from "./schedule";
import { assertUrlAllowed, isBlockedHostname, isBlockedIP } from "./ssrf";
import { slugify } from "./ids";
const page = (price: string, year: string, extra = "") => `API Pricing · Acme
API Pricing
Input: ${price} / million tokens
Output: $30 / million tokens
${extra}
`;
describe("canonicalization", () => {
it("ignores render noise (scripts, nav, copyright year, counters, tracking params)", () => {
const a = canonicalizeHtml(page("$10", "2025"), "https://acme.com/pricing");
const b = canonicalizeHtml(page("$10", "2026"), "https://acme.com/pricing");
expect(a.canonicalHash).toBe(b.canonicalHash);
expect(a.rawHash).not.toBe(b.rawHash);
expect(a.title).toBe("API Pricing · Acme");
expect(a.headings).toEqual(["API Pricing"]);
expect(stripTrackingParams("https://x.com/a?utm_source=t&id=3&fbclid=9#frag")).toBe("https://x.com/a?id=3");
});
it("changes the canonical hash when content changes", () => {
const a = canonicalizeHtml(page("$10", "2026"));
const b = canonicalizeHtml(page("$8", "2026"));
expect(a.canonicalHash).not.toBe(b.canonicalHash);
expect(hammingHex(a.semanticHash, b.semanticHash)).toBeLessThan(12);
});
it("normalizes plain text", () => {
const t = canonicalizeText("a b\r\n\r\n c \n");
expect(t.text).toBe("a b\nc");
});
});
describe("diff engines", () => {
it("pairs modified lines", () => {
const d = diffText("Input: $10 / million tokens\nOutput: $30\nUnchanged", "Input: $8 / million tokens\nOutput: $30\nUnchanged");
expect(d.modified).toEqual([{ before: "Input: $10 / million tokens", after: "Input: $8 / million tokens" }]);
expect(d.added).toEqual([]);
expect(d.stats.unchangedRatio).toBeGreaterThanOrEqual(0.5);
});
it("diffs JSON structurally", () => {
const d = diffJson({ price: { input: 10, output: 30 }, models: ["a"] }, { price: { input: 8, output: 30 }, models: ["a", "b"], region: "eu" });
expect(d.changes).toEqual(expect.arrayContaining([{ path: "price.input", op: "replace", before: 10, after: 8 }, { path: "region", op: "add", after: "eu" }]));
});
it("diffs keyed lists", () => {
const d = diffList([{ key: "1", title: "A", status: "open" }, { key: "2", title: "B" }], [{ key: "1", title: "A", status: "resolved" }, { key: "3", title: "C" }], ["status"]);
expect(d.added.map((i) => i.key)).toEqual(["3"]);
expect(d.removed.map((i) => i.key)).toEqual(["2"]);
expect(d.modified[0]).toMatchObject({ key: "1", fields: ["status"] });
expect(summarizeDiff(d)).toMatchObject({ kind: "list", counts: { added: 1, removed: 1, modified: 1 } });
});
});
describe("heuristics", () => {
it("scores a copyright bump as noise", () => {
const d = diffText("© 2025 Acme\nAll rights reserved", "© 2026 Acme\nAll rights reserved");
const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com", sourceCategories: [] });
expect(h.signal).toBeLessThan(0.1);
expect(h.eventType).toBe("unknown");
});
it("detects a pricing change with extracted facts", () => {
const d = diffText("Input: $10 / million tokens", "Input: $8 / million tokens");
const h = evaluateChange(d, { sensorType: "HTML", url: "https://acme.com/pricing", sourceCategories: ["ai"] });
expect(h.eventType).toBe("pricing_change");
expect(h.signal).toBeGreaterThan(0.5);
expect(h.facts[0]).toMatchObject({ kind: "price", before: "$10 / million tokens", after: "$8 / million tokens" });
const desc = describeChange(h, d, { sourceName: "Acme", url: "https://acme.com/pricing", sensorName: "pricing" });
expect(desc.title).toContain("$10 / million tokens → $8 / million tokens");
});
it("classifies new status incidents and CVEs", () => {
const inc = diffList([], [{ key: "incident:1", title: "Elevated error rates — investigating", summary: "We are investigating elevated error rates on the API." }]);
expect(evaluateChange(inc, { sensorType: "STATUSPAGE", url: "https://status.acme.com/api/v2/summary.json", sourceCategories: ["ai"] }).eventType).toBe("incident");
const cve = diffList([], [{ key: "CVE-2026-1234", title: "CVE-2026-1234 — Acme Router: Remote Code Execution Vulnerability", summary: "Acme Router contains an RCE vulnerability." }]);
expect(evaluateChange(cve, { sensorType: "REST_API", url: "https://kev.example", sourceCategories: ["cyber"] }).eventType).toBe("vulnerability");
});
});
describe("scoring", () => {
it("weights components and stays in range", () => {
const r = computeImportance({ eventType: "pricing_change", sourceImportance: 92, entityImportance: 90, novelty: 90, magnitude: 60, confirmations: 1 });
expect(r.score).toBeGreaterThan(75);
expect(r.score).toBeLessThanOrEqual(100);
expect(Object.keys(r.components)).toHaveLength(8);
const low = computeImportance({ eventType: "content_change", sourceImportance: 35, entityImportance: 30, novelty: 10, magnitude: 5, confirmations: 0 });
expect(low.score).toBeLessThan(35);
expect(sourceImportanceFromTier("S")).toBe(92);
expect(computeConfidence({ sourceAuthenticity: 1, extraction: 1, diffClarity: 1, structured: true, confirmations: 2, llmAgreement: 1 })).toBe(100);
});
});
describe("adaptive schedule", () => {
it("tightens after recent changes and backs off after errors", () => {
const base = { tier: "B" as const, changes7d: 0, events7d: 0, consecutiveErrors: 0, lastWas304: false };
const quiet = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 30 * 86400e3) });
const burst = nextIntervalSeconds({ ...base, lastChangeAt: new Date(Date.now() - 5 * 60e3), changes7d: 30 });
const failing = nextIntervalSeconds({ ...base, lastChangeAt: null, consecutiveErrors: 4 });
expect(burst).toBeLessThan(quiet);
expect(burst).toBeGreaterThanOrEqual(300);
expect(failing).toBeGreaterThan(quiet);
});
});
describe("ssrf policy", () => {
it("blocks private ranges, metadata and cluster hosts", async () => {
expect(isBlockedIP("10.1.2.3")).toBe(true);
expect(isBlockedIP("169.254.169.254")).toBe(true);
expect(isBlockedIP("100.64.0.1")).toBe(true);
expect(isBlockedIP("::ffff:192.168.1.1")).toBe(true);
expect(isBlockedIP("8.8.8.8")).toBe(false);
// NAT64: the embedded IPv4 decides (64:ff9b::8c52:7004 = 140.82.112.4 public; 64:ff9b::a00:1 = 10.0.0.1 private)
expect(isBlockedIP("64:ff9b::8c52:7004")).toBe(false);
expect(isBlockedIP("64:ff9b::a00:1")).toBe(true);
expect(isBlockedIP("64:ff9b::a9fe:a9fe")).toBe(true);
expect(isBlockedHostname("m3u96a.maclustr.io")).toBe(true);
expect(isBlockedHostname("localhost")).toBe(true);
await expect(assertUrlAllowed("http://127.0.0.1:8080/")).rejects.toThrow(/not allowed/);
await expect(assertUrlAllowed("ftp://example.com")).rejects.toThrow(/Scheme/);
await expect(assertUrlAllowed("https://user:pw@example.com")).rejects.toThrow(/Credentials/);
});
});
describe("hashing", () => {
it("shingles + jaccard measure similarity", () => {
const a = shingles("OpenAI releases new model GPT-5 with lower pricing");
const b = shingles("OpenAI releases new model GPT-5 with lower prices today");
const c = shingles("FDA approves a new drug for diabetes");
expect(jaccard(a, b)).toBeGreaterThan(0.3);
expect(jaccard(a, c)).toBe(0);
expect(simhash("x")).toHaveLength(16);
expect(slugify("Anthropic: Claude Opus 5 — pricing changed!")).toBe("anthropic-claude-opus-5-pricing-changed");
});
});
describe("connector-class heuristic priors", () => {
it("classifies TLS/DNS json changes and EDGAR/package/openapi list items", () => {
const jsonDiff = { kind: "json" as const, changes: [{ path: "issuer.O", before: "Let's Encrypt", after: "DigiCert Inc", op: "changed" as const }], stats: { added: 0, removed: 0, modified: 1, unchangedRatio: 0.9 } };
expect(evaluateChange(jsonDiff as never, { sensorType: "TLS", url: "tls://www.example.com", sourceCategories: ["technology"] }).eventType).toBe("certificate_change");
expect(evaluateChange(jsonDiff as never, { sensorType: "DNS", url: "dns://example.com", sourceCategories: ["technology"] }).eventType).toBe("dns_change");
const filing = { kind: "list" as const, added: [{ key: "0001-26-1", form: "8-K", items: "2.02,9.01", title: "Acme: 8-K — Current report (2.02)", summary: "Results of operations", url: "https://www.sec.gov/x" }], removed: [], modified: [], stats: { added: 1, removed: 0, modified: 0, unchangedRatio: 0.98 } };
expect(evaluateChange(filing as never, { sensorType: "REST_API", url: "https://data.sec.gov/submissions/CIK0000000001.json", sourceCategories: ["technology"] }).eventType).toBe("earnings");
const pkg = { kind: "list" as const, added: [{ key: "19.3.0", version: "19.3.0", prerelease: false, title: "react 19.3.0", url: "https://www.npmjs.com/package/react/v/19.3.0" }], removed: [], modified: [{ before: { key: "latest", version: "19.2.0" }, after: { key: "latest", version: "19.3.0", prerelease: false, title: "react latest → 19.3.0" } }], stats: { added: 1, removed: 0, modified: 1, unchangedRatio: 0.95 } };
expect(evaluateChange(pkg as never, { sensorType: "REST_API", url: "https://www.npmjs.com/package/react", sourceCategories: ["open-source"] }).eventType).toBe("software_release");
const api = { kind: "list" as const, added: [{ key: "POST /v1/agents", fingerprint: "abc", deprecated: false, title: "POST /v1/agents", url: "https://x" }], removed: [], modified: [], stats: { added: 1, removed: 0, modified: 0, unchangedRatio: 0.99 } };
expect(evaluateChange(api as never, { sensorType: "JSON", url: "https://raw.githubusercontent.com/x/openapi.json", sourceCategories: ["ai"] }).eventType).toBe("api_change");
const robots = { kind: "text" as const, added: ["User-agent: GPTBot", "Disallow: /"], removed: [], modified: [], stats: { added: 2, removed: 0, modified: 0, unchangedRatio: 0.8 } };
const r = evaluateChange(robots as never, { sensorType: "FILE", url: "https://www.example.com/robots.txt", sourceCategories: ["media"] });
expect(r.eventType).toBe("crawler_policy_change");
expect(r.reasons).toContain("AI crawler directive changed");
});
});