import type { CandidateKind, DeepCandidate } from "@websensor/connectors"; import { factoryConfig } from "../config"; /** * Candidate scoring (Source Factory, stage "signal/noise evaluation before shadow"). Combines the discovery * value (evidence quality) with the organization's importance, the class of page, first-party confidence, * expected change frequency and fetch cost. Explainable: every score carries its reasons. */ export interface ScoredCandidate { cand: DeepCandidate; score: number; reasons: string[]; } /** How much a page class is worth for change intelligence (1 = neutral). */ export const KIND_WEIGHT: Record = { status: 1.15, security: 1.15, press: 1.1, news: 1.05, filings: 1.1, changelog: 1.05, releases: 1.0, pricing: 1.05, models: 1.05, ir: 0.95, legal: 0.9, blog: 0.9, api: 0.9, leadership: 0.85, data: 0.85, sitemap: 0.8, docs: 0.7, careers: 0.6, posture: 0.6, other: 0.7, }; export function scoreCandidate(c: DeepCandidate, seed: { importance: number; weight?: number }): ScoredCandidate { const reasons: string[] = []; let score = c.value; reasons.push(`evidence ${c.value.toFixed(2)} (${c.evidence})`); const kw = KIND_WEIGHT[c.kind] ?? 0.7; score *= kw; if (kw !== 1) reasons.push(`${c.kind} ×${kw.toFixed(2)}`); const imp = seed.importance === 3 ? 1.1 : seed.importance === 1 ? 0.85 : 1; score *= imp * (seed.weight ?? 1); if (imp !== 1) reasons.push(`organization importance ${seed.importance} ×${imp.toFixed(2)}`); score *= c.firstPartyConfidence; if (c.firstPartyConfidence < 1) reasons.push(`first-party confidence ${c.firstPartyConfidence}`); if (c.itemsPerDay !== null && c.itemsPerDay !== undefined) { if (c.itemsPerDay > 60) { score *= 0.85; reasons.push(`firehose ${c.itemsPerDay}/day ×0.85`); } else if (c.itemsPerDay < 0.02) { score *= 0.8; reasons.push(`nearly dormant ${c.itemsPerDay}/day ×0.80`); } else if (c.itemsPerDay >= 0.2) { score *= 1.05; reasons.push(`active ${c.itemsPerDay}/day ×1.05`); } } if (c.fetchCost.ms > 8000) { score -= 0.08; reasons.push(`slow (${Math.round(c.fetchCost.ms)} ms) −0.08`); } if (c.fetchCost.bytes > 5 * 1024 * 1024) { score -= 0.08; reasons.push(`heavy (${Math.round(c.fetchCost.bytes / 1048576)} MB) −0.08`); } score = Math.round(Math.max(0, Math.min(1.2, score)) * 1000) / 1000; return { cand: c, score, reasons }; } /** Promotion policy: caps per organization and per class, best score first. Returns the selected candidates and the reason for the others. */ export function selectForShadow(scored: ScoredCandidate[], existing: { kinds: Map; total: number }): { selected: ScoredCandidate[]; skipped: { s: ScoredCandidate; reason: string }[] } { const caps = factoryConfig.caps; const groupOf = (k: CandidateKind): keyof typeof caps => (k === "news" || k === "press" || k === "blog" || k === "ir" || k === "changelog" || k === "security" ? "feeds" : k === "pricing" || k === "legal" || k === "careers" || k === "leadership" || k === "docs" || k === "data" || k === "other" ? "pages" : (k as keyof typeof caps)); const used = new Map(); for (const [k, n] of existing.kinds) used.set(groupOf(k), (used.get(groupOf(k)) ?? 0) + n); let total = existing.total; const selected: ScoredCandidate[] = []; const skipped: { s: ScoredCandidate; reason: string }[] = []; const legal = (existing.kinds.get("legal") ?? 0); let legalUsed = legal; for (const s of [...scored].sort((a, b) => b.score - a.score)) { if (s.score < factoryConfig.minScore) { skipped.push({ s, reason: `score ${s.score} < ${factoryConfig.minScore}` }); continue; } if (total >= caps.total) { skipped.push({ s, reason: `organization cap ${caps.total} reached` }); continue; } // Feed-type candidates (rss connector) and HTML pages are capped by group; connectors with their own cap by kind. const g = s.cand.connector === "rss" ? "feeds" : s.cand.connector === "http" && s.cand.kind !== "posture" ? "pages" : groupOf(s.cand.kind); const cap = caps[g as keyof typeof caps] ?? 1; if ((used.get(g) ?? 0) >= cap) { skipped.push({ s, reason: `${g} cap ${cap} reached` }); continue; } if (s.cand.kind === "legal") { if (legalUsed >= caps.legal) { skipped.push({ s, reason: `legal cap ${caps.legal} reached` }); continue; } legalUsed++; } used.set(g, (used.get(g) ?? 0) + 1); total++; selected.push(s); } return { selected, skipped }; }