TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { describe, expect, it } from "vitest";2import { coverageKey, registrableDomain } from "@websensor/core";3import type { DeepCandidate } from "@websensor/connectors";4import { itemsPerDayOf } from "@websensor/connectors";5import { KIND_WEIGHT, scoreCandidate, selectForShadow } from "./score";67const cand = (over: Partial<DeepCandidate>): DeepCandidate => ({ url: "https://example.org/feed", type: "RSS", connector: "rss", kind: "news", name: "news feed", config: {}, suggestedTier: "B", evidence: "well-known path", value: 0.8, itemsPerDay: 0.5, firstPartyConfidence: 1, fetchCost: { bytes: 20_000, ms: 400 }, ...over });89describe("registrable domain (coverage key)", () => {10 it("collapses hosts to the organization's registrable domain", () => {11 expect(registrableDomain("ir.apple.com")).toBe("apple.com");12 expect(registrableDomain("www.ons.gov.uk")).toBe("ons.gov.uk");13 expect(registrableDomain("www.bankofengland.co.uk")).toBe("bankofengland.co.uk");14 expect(registrableDomain("statcan.gc.ca")).toBe("statcan.gc.ca");15 expect(registrableDomain("open.canada.ca")).toBe("open.canada.ca"); // federal departments are distinct organizations16 expect(registrableDomain("www.canada.ca")).toBe("canada.ca");17 expect(registrableDomain("boj.or.jp")).toBe("boj.or.jp");18 expect(coverageKey("https://WWW.Example.COM/path")).toBe("example.com");19 });20});2122describe("factory scoring", () => {23 it("weights page classes and organization importance", () => {24 const status = scoreCandidate(cand({ kind: "status", connector: "statuspage", value: 0.95, itemsPerDay: null }), { importance: 2 });25 const careers = scoreCandidate(cand({ kind: "careers", connector: "http", value: 0.35, itemsPerDay: null }), { importance: 2 });26 expect(status.score).toBeGreaterThan(careers.score);27 expect(KIND_WEIGHT.status).toBeGreaterThan(KIND_WEIGHT.careers);28 const big = scoreCandidate(cand({}), { importance: 3 });29 const small = scoreCandidate(cand({}), { importance: 1 });30 expect(big.score).toBeGreaterThan(small.score);31 expect(big.reasons.some((r) => /importance 3/.test(r))).toBe(true);32 });33 it("penalises firehoses, dormant feeds and slow endpoints, explainably", () => {34 const base = scoreCandidate(cand({}), { importance: 2 }).score;35 expect(scoreCandidate(cand({ itemsPerDay: 200 }), { importance: 2 }).score).toBeLessThan(base);36 expect(scoreCandidate(cand({ itemsPerDay: 0.001 }), { importance: 2 }).score).toBeLessThan(base);37 const slow = scoreCandidate(cand({ fetchCost: { bytes: 100, ms: 12_000 } }), { importance: 2 });38 expect(slow.score).toBeLessThan(base);39 expect(slow.reasons.some((r) => /slow/.test(r))).toBe(true);40 });41 it("applies per-organization caps and the minimum score", () => {42 const feeds = Array.from({ length: 6 }, (_, i) => scoreCandidate(cand({ url: `https://example.org/f${i}`, value: 0.9 - i * 0.02 }), { importance: 2 }));43 const low = scoreCandidate(cand({ url: "https://example.org/careers", kind: "careers", connector: "http", value: 0.2, itemsPerDay: null }), { importance: 1 });44 const legal = ["terms", "privacy", "cookies"].map((p) => scoreCandidate(cand({ url: `https://example.org/${p}`, kind: "legal", connector: "http", type: "HTML", value: 0.7, itemsPerDay: null }), { importance: 2 }));45 const { selected, skipped } = selectForShadow([...feeds, low, ...legal], { kinds: new Map(), total: 0 });46 expect(selected.filter((s) => s.cand.connector === "rss")).toHaveLength(4);47 expect(selected.filter((s) => s.cand.kind === "legal")).toHaveLength(2);48 expect(skipped.some((s) => s.s === low && /score/.test(s.reason))).toBe(true);49 expect(skipped.some((s) => /feeds cap/.test(s.reason))).toBe(true);50 });51 it("respects sensors the organization already has", () => {52 const feed = scoreCandidate(cand({}), { importance: 2 });53 const { selected, skipped } = selectForShadow([feed], { kinds: new Map([["news", 4]]), total: 4 });54 expect(selected).toHaveLength(0);55 expect(skipped[0]?.reason).toMatch(/feeds cap/);56 });57});5859describe("expected change frequency", () => {60 it("uses the median gap and survives one bogus date", () => {61 const day = 86400e3;62 const now = Date.UTC(2026, 8, 13);63 const dates = [now, now - day, now - 2 * day, now - 3 * day, now - 4 * day, now - 5000 * day];64 expect(itemsPerDayOf(dates)).toBe(1);65 expect(itemsPerDayOf([now, now - day])).toBeNull();66 expect(itemsPerDayOf([now, now - 30 * day, now - 60 * day, now - 90 * day])).toBeCloseTo(0.033, 2);67 });68});69