TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import type { CandidateKind, DeepCandidate } from "@websensor/connectors";2import { factoryConfig } from "../config";34/**5 * Candidate scoring (Source Factory, stage "signal/noise evaluation before shadow"). Combines the discovery6 * value (evidence quality) with the organization's importance, the class of page, first-party confidence,7 * expected change frequency and fetch cost. Explainable: every score carries its reasons.8 */9export interface ScoredCandidate {10 cand: DeepCandidate;11 score: number;12 reasons: string[];13}1415/** How much a page class is worth for change intelligence (1 = neutral). */16export const KIND_WEIGHT: Record<CandidateKind, number> = {17 status: 1.15,18 security: 1.15,19 press: 1.1,20 news: 1.05,21 filings: 1.1,22 changelog: 1.05,23 releases: 1.0,24 pricing: 1.05,25 models: 1.05,26 ir: 0.95,27 legal: 0.9,28 blog: 0.9,29 api: 0.9,30 leadership: 0.85,31 data: 0.85,32 sitemap: 0.8,33 docs: 0.7,34 careers: 0.6,35 posture: 0.6,36 other: 0.7,37};3839export function scoreCandidate(c: DeepCandidate, seed: { importance: number; weight?: number }): ScoredCandidate {40 const reasons: string[] = [];41 let score = c.value;42 reasons.push(`evidence ${c.value.toFixed(2)} (${c.evidence})`);43 const kw = KIND_WEIGHT[c.kind] ?? 0.7;44 score *= kw;45 if (kw !== 1) reasons.push(`${c.kind} ×${kw.toFixed(2)}`);46 const imp = seed.importance === 3 ? 1.1 : seed.importance === 1 ? 0.85 : 1;47 score *= imp * (seed.weight ?? 1);48 if (imp !== 1) reasons.push(`organization importance ${seed.importance} ×${imp.toFixed(2)}`);49 score *= c.firstPartyConfidence;50 if (c.firstPartyConfidence < 1) reasons.push(`first-party confidence ${c.firstPartyConfidence}`);51 if (c.itemsPerDay !== null && c.itemsPerDay !== undefined) {52 if (c.itemsPerDay > 60) {53 score *= 0.85;54 reasons.push(`firehose ${c.itemsPerDay}/day ×0.85`);55 } else if (c.itemsPerDay < 0.02) {56 score *= 0.8;57 reasons.push(`nearly dormant ${c.itemsPerDay}/day ×0.80`);58 } else if (c.itemsPerDay >= 0.2) {59 score *= 1.05;60 reasons.push(`active ${c.itemsPerDay}/day ×1.05`);61 }62 }63 if (c.fetchCost.ms > 8000) {64 score -= 0.08;65 reasons.push(`slow (${Math.round(c.fetchCost.ms)} ms) −0.08`);66 }67 if (c.fetchCost.bytes > 5 * 1024 * 1024) {68 score -= 0.08;69 reasons.push(`heavy (${Math.round(c.fetchCost.bytes / 1048576)} MB) −0.08`);70 }71 score = Math.round(Math.max(0, Math.min(1.2, score)) * 1000) / 1000;72 return { cand: c, score, reasons };73}7475/** Promotion policy: caps per organization and per class, best score first. Returns the selected candidates and the reason for the others. */76export function selectForShadow(scored: ScoredCandidate[], existing: { kinds: Map<CandidateKind, number>; total: number }): { selected: ScoredCandidate[]; skipped: { s: ScoredCandidate; reason: string }[] } {77 const caps = factoryConfig.caps;78 const groupOf = (k: CandidateKind): keyof typeof caps => (k === "news" || k === "press" || k === "blog" || k === "ir" || k === "changelog" || k === "security" ? "feeds" : k === "pricing" || k === "legal" || k === "careers" || k === "leadership" || k === "docs" || k === "data" || k === "other" ? "pages" : (k as keyof typeof caps));79 const used = new Map<string, number>();80 for (const [k, n] of existing.kinds) used.set(groupOf(k), (used.get(groupOf(k)) ?? 0) + n);81 let total = existing.total;82 const selected: ScoredCandidate[] = [];83 const skipped: { s: ScoredCandidate; reason: string }[] = [];84 const legal = (existing.kinds.get("legal") ?? 0);85 let legalUsed = legal;86 for (const s of [...scored].sort((a, b) => b.score - a.score)) {87 if (s.score < factoryConfig.minScore) {88 skipped.push({ s, reason: `score ${s.score} < ${factoryConfig.minScore}` });89 continue;90 }91 if (total >= caps.total) {92 skipped.push({ s, reason: `organization cap ${caps.total} reached` });93 continue;94 }95 // Feed-type candidates (rss connector) and HTML pages are capped by group; connectors with their own cap by kind.96 const g = s.cand.connector === "rss" ? "feeds" : s.cand.connector === "http" && s.cand.kind !== "posture" ? "pages" : groupOf(s.cand.kind);97 const cap = caps[g as keyof typeof caps] ?? 1;98 if ((used.get(g) ?? 0) >= cap) {99 skipped.push({ s, reason: `${g} cap ${cap} reached` });100 continue;101 }102 if (s.cand.kind === "legal") {103 if (legalUsed >= caps.legal) {104 skipped.push({ s, reason: `legal cap ${caps.legal} reached` });105 continue;106 }107 legalUsed++;108 }109 used.set(g, (used.get(g) ?? 0) + 1);110 total++;111 selected.push(s);112 }113 return { selected, skipped };114}115