TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import type { DiffResult } from "./diff";2import { EVENT_TYPES, eventTypeSpec } from "./taxonomy";34/**5 * Stage-1 interpretation: cheap deterministic rules that (a) filter noise so we never6 * spend an LLM call on a copyright-year bump and (b) propose an event type, a magnitude7 * and keywords. The LLM stage (optional) refines title/summary/type for candidates that8 * clear the bar.9 */1011export interface HeuristicResult {12 /** 0–1: how likely the change is meaningful (1 = clearly meaningful). */13 signal: number;14 /** Fraction of changed lines classified as noise. */15 noiseRatio: number;16 eventType: string;17 /** 0–100 magnitude of the change. */18 magnitude: number;19 keywords: string[];20 /** Human-readable hints explaining the decision (auditable). */21 reasons: string[];22 /** Extracted money/percent/version facts (before → after). */23 facts: { kind: "price" | "percent" | "version" | "number" | "date"; before?: string; after?: string }[];24}2526const NOISE_LINE = [27 /^©|\bcopyright\b|\ball rights reserved\b/i,28 /^\d{4}$/, // a bare year29 /\b(20\d{2})\b.*\b(20\d{2})\b/, // year ranges in footers30 /^(home|menu|search|login|sign in|sign up|subscribe|share|print|back to top|skip to (main )?content|close|next|previous|read more|learn more)$/i,31 /^\s*[\d,.]+\s*$/, // bare numbers (counters)32 /\b(views?|likes?|shares?|comments?|followers?)\b\s*:?\s*[\d,.]+/i,33 /\bcookie|consent|privacy preferences|accept all\b/i,34 /^(loading|please wait)\b/i,35 /\b(posted|published|updated)\s+\d+\s+(seconds?|minutes?|hours?|days?)\s+ago\b/i,36];3738const TYPE_RULES: { type: string; re: RegExp; weight: number }[] = [39 { type: "pricing_change", re: /\b(price|pricing|per (million|1k|1m) tokens|\$\s?\d|€\s?\d|£\s?\d|usd|per month|per seat|per user|\/mo\b|billing|discount|free tier|rate card)\b/i, weight: 1 },40 { type: "model_release", re: /\b(new model|model release|introducing (gpt|claude|gemini|llama|mistral|grok|qwen|deepseek|o\d)|frontier model|foundation model|llm|multimodal model|parameters|context window|benchmark)\b/i, weight: 0.9 },41 { type: "security_advisory", re: /\b(security advisory|security bulletin|advisory|patch(ed)?|exploit(ed|ation)?|zero[- ]day|mitigation|remote code execution|privilege escalation|hotfix)\b/i, weight: 1 },42 { type: "vulnerability", re: /\b(cve-\d{4}-\d{4,}|cvss|vulnerabilit(y|ies)|known exploited|kev catalog)\b/i, weight: 1.1 },43 { type: "breach", re: /\b(data breach|breach(ed)?|compromised|unauthori[sz]ed access|leak(ed)?|exfiltrat)/i, weight: 1.1 },44 { type: "outage", re: /\b(outage|major outage|service disruption|unavailable|downtime|degraded performance|partial outage)\b/i, weight: 1 },45 { type: "incident", re: /\b(incident|investigating|identified|monitoring|resolved|postmortem|post-mortem|root cause)\b/i, weight: 0.8 },46 { type: "maintenance", re: /\b(scheduled maintenance|maintenance window|planned maintenance)\b/i, weight: 0.9 },47 { type: "recall", re: /\b(recall(s|ed)?|safety notice|stop sale|do not use)\b/i, weight: 1 },48 { type: "drug_approval", re: /\b(fda approv|approval|approved|authoriz(ed|ation)|indication|biologics license|new drug application|nda|bla|marketing authori[sz]ation|emergency use)\b/i, weight: 0.8 },49 { type: "clinical_trial", re: /\b(phase (1|2|3|i|ii|iii)|clinical trial|topline results|primary endpoint|enrollment|study results)\b/i, weight: 0.9 },50 { type: "regulatory_filing", re: /\b(filing|filed|rulemaking|proposed rule|final rule|notice of|federal register|docket|comment period|enforcement action|consent order)\b/i, weight: 0.8 },51 { type: "financial_filing", re: /\b(10-k|10-q|8-k|s-1|form 4|13f|6-k|20-f|prospectus|proxy statement|def 14a)\b/i, weight: 1 },52 { type: "earnings", re: /\b(earnings|quarterly results|fiscal (q[1-4]|quarter|year)|revenue (grew|increased|declined|of)|eps|guidance|net income)\b/i, weight: 1 },53 { type: "monetary_policy", re: /\b(interest rate|policy rate|rate decision|basis points|fomc|monetary policy|overnight rate|bank rate|quantitative)\b/i, weight: 1 },54 { type: "economic_release", re: /\b(cpi|consumer price index|inflation|unemployment rate|nonfarm payroll|gdp|gross domestic product|retail sales|labour force survey|labor force|housing starts|trade balance|producer price)\b/i, weight: 0.9 },55 { type: "acquisition", re: /\b(acqui(re|red|sition)|to acquire|merger|merge with|takeover|buyout)\b/i, weight: 1 },56 { type: "funding", re: /\b(series [a-h]\b|raised \$|funding round|seed round|valuation of|investment of \$)/i, weight: 0.9 },57 { type: "partnership", re: /\b(partnership|partners? with|collaboration with|teams? up with|joint venture|strategic alliance)\b/i, weight: 0.8 },58 { type: "leadership_change", re: /\b(appoint(s|ed|ment)|named (as )?(ceo|cfo|cto|coo|president|chair)|steps? down|resign(s|ed|ation)|chief (executive|financial|technology|operating) officer|board of directors|joins as)\b/i, weight: 0.9 },59 { type: "layoffs", re: /\b(layoffs?|laid off|job cuts|workforce reduction|restructuring|reduction in force|redundanc)/i, weight: 1 },60 { type: "job_expansion", re: /\b(we're hiring|now hiring|open (roles|positions)|careers?|job openings?)\b/i, weight: 0.5 },61 { type: "product_launch", re: /\b(introducing|launch(es|ed|ing)?|now available|available today|announc(es|ed|ing)|unveil(s|ed)|debuts?|general availability|ga release|new product)\b/i, weight: 0.8 },62 { type: "software_release", re: /\b(v?\d+\.\d+(\.\d+)?(-[a-z0-9.]+)?\b.*\b(release|released|changelog|patch notes|what's new)|release notes|version \d|stable release|lts\b|beta release|rc\d)/i, weight: 0.8 },63 { type: "repository_release", re: /\b(github release|tag v?\d|pre-release|assets? \d+)\b/i, weight: 0.6 },64 { type: "API_change", re: /\b(api|endpoint|sdk|deprecat(ed|ion)|sunset|breaking change|rate limit|quota|webhook|graphql|rest api|openapi|parameter)\b/i, weight: 0.8 },65 { type: "policy_change", re: /\b(policy|policies|usage policy|acceptable use|guidelines|code of conduct|content policy|privacy policy)\b/i, weight: 0.8 },66 { type: "terms_change", re: /\b(terms of (service|use)|terms and conditions|service agreement|end user license|eula|licen[cs]e terms)\b/i, weight: 0.9 },67 { type: "availability_change", re: /\b(discontinued|end of life|end-of-life|eol\b|no longer (available|supported)|retir(ed|ing)|sunset|coming soon|waitlist|now in (beta|preview)|preview|region(s)? available|expands? to)\b/i, weight: 0.8 },68 { type: "new_region", re: /\b(new region|region launch|data center|datacenter|availability zone|now available in (europe|asia|canada|australia|india|japan|brazil|uk|us-)|opens? (in|new office))\b/i, weight: 0.8 },69 { type: "dataset_release", re: /\b(dataset|data release|open data|benchmark suite|corpus)\b/i, weight: 0.7 },70 { type: "standard_update", re: /\b(rfc \d+|w3c recommendation|candidate recommendation|working draft|specification|standard(s)? (update|published)|editor's draft)\b/i, weight: 0.8 },71 { type: "scientific_publication", re: /\b(paper|preprint|arxiv|peer[- ]reviewed|published in (nature|science|cell|lancet|nejm)|doi:|abstract)\b/i, weight: 0.7 },72 { type: "government_announcement", re: /\b(minister|ministry|prime minister|president|secretary|department of|government of|executive order|statement by|press release|backgrounder)\b/i, weight: 0.6 },73 { type: "legal_change", re: /\b(antitrust|regulation|statute|compliance deadline|injunction)\b/i, weight: 0.7 },74 { type: "documentation_change", re: /\b(docs?|documentation|guide|tutorial|reference|quickstart|readme|faq|how to)\b/i, weight: 0.5 },75 // ---- 2026-09-11: cyber classes ----76 { type: "zero_day", re: /\b(zero[- ]day|0[- ]day|0day|in[- ]the[- ]wild|itw\b|unpatched (vulnerability|flaw|bug)|no patch (is )?available)\b/i, weight: 1.4 },77 { type: "active_exploitation", re: /\b(actively exploited|active exploitation|exploited in the wild|known exploited|kev catalog|added to (the )?kev|exploitation (has been )?(observed|detected|confirmed)|under active attack|mass exploitation)\b/i, weight: 1.4 },78 { type: "patch_release", re: /\b(patch tuesday|security (update|patch|fix)(es)? (released|available)|patches? (for|address(es|ing))|hotfix|out[- ]of[- ]band (update|patch)|fixed in version|update now|cumulative update)\b/i, weight: 1 },79 { type: "supply_chain_attack", re: /\b(supply[- ]chain (attack|compromise|incident)|malicious (package|npm|pypi|dependency|commit|update)|typosquat|compromised (package|build|pipeline|dependency|maintainer)|backdoored|trojanized|dependency confusion|xz[- ]utils|solarwinds-style)\b/i, weight: 1.4 },80 { type: "credential_leak", re: /\b(credential(s)? (leak|dump|exposed|stolen)|leaked (credentials|passwords|api keys|tokens|secrets)|password (dump|leak)|exposed (api key|secret|token)|infostealer logs|combolist)\b/i, weight: 1.3 },81 { type: "malware_campaign", re: /\b(malware (campaign|family|strain|loader)|phishing campaign|spear[- ]phishing|infostealer|botnet|trojan|backdoor|rootkit|rat\b|command[- ]and[- ]control|c2 (server|infrastructure)|apt\d{1,3}\b|threat actor|attributed to)\b/i, weight: 1.1 },82 { type: "ransomware", re: /\b(ransomware|ransom (note|demand|payment)|double extortion|leak site|encrypted (their|its|the) (files|systems|network)|lockbit|alphv|blackcat|cl0p|clop|akira|qilin|ransomhub|play ransomware|medusa|black basta|hunters international)\b/i, weight: 1.3 },83 // ---- finance ----84 { type: "merger", re: /\b(merger|merge with|merger agreement|combination of|combined company|all-stock (deal|transaction))\b/i, weight: 1 },85 { type: "ipo", re: /\b(ipo|initial public offering|goes public|going public|listing on (the )?(nyse|nasdaq|tsx|lse)|priced its (initial )?public offering|direct listing|spac merger|de-spac)\b/i, weight: 1.1 },86 { type: "guidance", re: /\b((raises|raised|lowers|lowered|cuts|cut|reaffirms|reaffirmed|updates|updated|withdraws|withdrew|maintains) (its )?(full[- ]year |fy\d* |annual |quarterly |fiscal )?(guidance|outlook|forecast)|outlook for (fiscal|fy|20\d\d)|expects (revenue|earnings|eps) (of|between|in the range))\b/i, weight: 1.1 },87 { type: "dividend", re: /\b(dividend|quarterly cash distribution|declares? (a )?(quarterly |special |annual )?(cash )?dividend|ex[- ]dividend|dividend increase)\b/i, weight: 1 },88 { type: "buyback", re: /\b(buyback|share repurchase|repurchase program|repurchase authorization|stock repurchase|normal course issuer bid|ncib)\b/i, weight: 1.1 },89 { type: "bankruptcy", re: /\b(bankruptcy|chapter 11|chapter 7|chapter 15|insolvency|insolvent|receivership|administration proceedings|ccaa\b|creditor protection|liquidation|wind[- ]down of operations|files? for bankruptcy)\b/i, weight: 1.3 },90 { type: "capital_raise", re: /\b(capital raise|raises? capital|equity offering|public offering of (common|ordinary) shares|private placement|bought deal|at-the-market offering|convertible (notes?|debentures?)|senior notes offering|debt offering|rights offering|bond issuance)\b/i, weight: 1 },91 { type: "rating_change", re: /\b((upgrades?|downgrades?|affirms?|places?|revises?) .{0,40}(rating|outlook)|credit rating|outlook (to )?(negative|positive|stable)|(moody's|s&p|fitch|dbrs|morningstar) (upgrade|downgrade|affirm|rating action)|investment grade|junk status|rating watch)\b/i, weight: 1.1 },92 { type: "insider_transaction", re: /\b(form 4|insider (buy|sell|purchase|sale|transaction)|10b5-1|beneficial ownership|sedi filing|statement of changes in beneficial ownership)\b/i, weight: 1 },93 // ---- government / legal ----94 { type: "regulatory_action", re: /\b(enforcement action|consent order|cease[- ]and[- ]desist|civil penalty|fine(d|s)? (of )?\$|settles? (charges|allegations)|charged with|order(s|ed) .{0,30} to pay|regulatory action|compliance order|penalt(y|ies) of|sanctioned by (the )?(sec|ftc|cftc|finra|fca|osc|amf))\b/i, weight: 1.1 },95 { type: "lawsuit", re: /\b(lawsuit|sues?|sued|files? (a )?(class[- ]action|complaint|suit)|litigation|plaintiff|defendant|alleges|allegations of|legal action against|antitrust (suit|case|complaint))\b/i, weight: 1 },96 { type: "court_decision", re: /\b(court (rules|ruled|ruling|decision|judgment|judgement|opinion|order)|supreme court|appeals? court|federal court|tribunal|judge (rules|ruled|orders|ordered|dismisses|dismissed)|verdict|injunction (granted|denied)|struck down|upheld|overturned|jugement|arrêt de la cour)\b/i, weight: 1.1 },97 { type: "sanction", re: /\b(sanctions?( list| designation| regime)?|designat(es|ed|ion) .{0,30}(sdn|ofac)|ofac|specially designated|asset freeze|export controls?|entity list|embargo|travel ban|blocked persons|special economic measures)\b/i, weight: 1.2 },98 { type: "legislation", re: /\b(bill c-\d+|bill s-\d+|h\.r\. ?\d+|s\. ?\d+\b|projet de loi|royal assent|third reading|second reading|first reading|introduced in (the )?(house|senate|parliament|assembly)|passed (the )?(house|senate|parliament)|signed into law|enacted|act, 20\d\d|directive \(eu\)|regulation \(eu\)|statutory instrument|order in council|décret|arrêté|loi n°)\b/i, weight: 1.1 },99 { type: "budget", re: /\b(budget (20\d\d|speech|plan|update|implementation)|fiscal update|economic statement|spending review|appropriations?|estimates|mise à jour économique|budget fédéral|budget provincial|deficit (of|projected)|surplus (of|projected))\b/i, weight: 1 },100 { type: "procurement", re: /\b(tender|request for proposals?|rfp\b|rfq\b|solicitation|contract award(ed)?|awarded (a )?(contract|tender)|procurement notice|appel d'offres|avis d'appel|invitation to bid|standing offer|supply arrangement)\b/i, weight: 1 },101 { type: "appointment", re: /\b(appoint(s|ed|ment of)|nominat(es|ed|ion of)|sworn in|takes? office|named (as )?(ambassador|judge|justice|commissioner|governor|deputy minister|chief of staff|director general|secretary general)|nomme|nomination de)\b/i, weight: 0.9 },102 { type: "emergency_notice", re: /\b(state of emergency|emergency (alert|declaration|order|measures)|evacuation (order|notice)|shelter in place|boil water (advisory|notice)|amber alert|tsunami (warning|alert)|hurricane (warning|watch)|tornado warning|wildfire (evacuation|alert)|flood warning|severe thunderstorm warning|red alert|alerte (rouge|d'urgence)|extreme cold warning|heat warning)\b/i, weight: 1.3 },103 // ---- health / science ----104 { type: "outbreak", re: /\b(outbreak|epidemic|pandemic|public health emergency|pheic|cases? (reported|confirmed) (of|in)|disease outbreak news|cluster of (cases|infections)|measles|cholera|ebola|marburg|mpox|h5n1|avian (flu|influenza)|dengue|listeria|salmonella|e\. ?coli|legionella|éclosion)\b/i, weight: 1.2 },105 { type: "drug_warning", re: /\b(boxed warning|black box warning|safety communication|drug safety (alert|update|communication)|medwatch|adverse (event|reaction)s? (report|warning)|label(ling)? change|contraindicat|dear healthcare provider|health professional risk communication|mise en garde)\b/i, weight: 1.2 },106 { type: "device_recall", re: /\b(device recall|medical device (recall|correction|removal)|class i recall|field safety notice|field safety corrective action|urgent medical device|510\(k\)|pma approval|de novo)\b/i, weight: 1.1 },107 { type: "retraction", re: /\b(retract(ed|ion|s)|expression of concern|withdrawn (paper|article|preprint)|erratum|errata|corrigendum|correction to)\b/i, weight: 1.2 },108 { type: "space_mission", re: /\b(launch (window|date|scrub|success)|liftoff|lifted off|payload|orbit|landing|splashdown|docking|undocking|spacewalk|eva\b|mission (update|success|failure)|rover|probe|satellite (launch|deploy)|iss\b|starship|falcon 9|ariane|vulcan|artemis|crew-\d+)\b/i, weight: 1 },109 // ---- transport ----110 { type: "accident", re: /\b(crash(ed|es)?|collision|derail(ment|ed)|accident|incident involving|emergency landing|runway (excursion|incursion)|fatalit(y|ies)|casualt(y|ies)|capsiz|sinking|explosion|mayday)\b/i, weight: 1.1 },111 { type: "grounding", re: /\b(ground(ed|ing) (the |its |all )?(fleet|aircraft|737|787|a320|a350|max)|emergency airworthiness directive|fleet[- ]wide inspection|stop[- ]sale order|stop[- ]delivery|halts? (deliveries|production|flights))\b/i, weight: 1.3 },112 { type: "safety_bulletin", re: /\b(airworthiness directive|safety (bulletin|directive|alert|recommendation|advisory)|service bulletin|civil aviation safety alert|notice to (airmen|air missions)|notam|safety information bulletin|marine safety (alert|bulletin))\b/i, weight: 1.1 },113 { type: "route_change", re: /\b(new (route|service|destination|direct flight|nonstop)|route (change|suspension|cancellation)|suspends? (flights|service|the route)|resumes? (flights|service)|adds? (flights|frequencies|a route)|schedule (change|adjustment)|seasonal service)\b/i, weight: 0.9 },114 { type: "production_delay", re: /\b(production (delay|halt|pause|slowdown|cut)|delivery delays?|delays? (deliveries|the launch|production|first flight|entry into service)|pushed (back )?to 20\d\d|postpone[ds]? (to|until)|supply (shortage|constraint)s?|chip shortage)\b/i, weight: 1 },115 // ---- product / service ----116 { type: "service_shutdown", re: /\b(shut(ting)? down|shutdown of|will (be )?(discontinued|retired|shut down|sunset|closed)|closing (the )?service|ceases? operations|end of service|service (ends|ending|termination)|sunsetting|is going away|no longer (offered|available|supported) (as of|after|starting))\b/i, weight: 1.1 },117 { type: "feature_removed", re: /\b(removed (the )?(feature|option|ability|support for)|feature (removed|removal|deprecated)|no longer supports?|dropping support|will remove|has been removed|removing (the|support))\b/i, weight: 1 },118 // ---- sports (official announcements only) ----119 { type: "sports_transaction", re: /\b(signs?|signed|re-signs?|trades?|traded|acquires? .{0,30} (in exchange|from the)|waive[ds]?|released (by|from) the (team|club)|claims? off waivers|extension|contract extension|recalls?|assigns? .{0,20} to (the )?(ahl|minors|g league)|placed on (the )?(injured|ir|il|reserve)|activated from|transfer (fee|window)|loan (deal|move)|free agent)\b/i, weight: 0.9 },120 { type: "sports_result", re: /\b(final score|defeats?|beats?|wins? (over|against)|loses? to|shutout|overtime|game \d+ (recap|preview)|standings|playoff (berth|clinch)|championship (game|final)|world series|stanley cup|super bowl|grand slam|podium|pole position|qualifying results)\b/i, weight: 0.7 },121 { type: "suspension", re: /\b(suspend(ed|s|sion)|banned|ban of|ineligible|fined \$|disciplinary (action|hearing)|anti[- ]doping (violation|rule)|provisional suspension|reprimand|sanctioned)\b/i, weight: 1 },122 { type: "schedule_change", re: /\b(postponed|rescheduled|schedule (change|update|released|announced)|kickoff time|puck drop|tip[- ]off|start time (changed|moved)|fixture (list|change)|weather delay|venue change|relocated to)\b/i, weight: 0.9 },123];124125const MONEY_RE = /(?:\$|€|£|usd|cad|eur)\s?\d[\d,]*(?:\.\d+)?(?:\s?(?:k|m|b|million|billion))?(?:\s?\/\s?(?:1k|1m|million|m)?\s?tokens?)?/gi;126const PERCENT_RE = /-?\d+(?:\.\d+)?\s?%/g;127const VERSION_RE = /\bv?\d+\.\d+(?:\.\d+){0,2}(?:-[a-z0-9.]+)?\b/gi;128const DATE_RE = /\b(?:\d{4}-\d{2}-\d{2}|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})\b/gi;129130function isNoiseLine(line: string): boolean {131 const l = line.trim();132 if (l.length < 3) return true;133 return NOISE_LINE.some((re) => re.test(l));134}135136function extractFacts(before: string, after: string): HeuristicResult["facts"] {137 const facts: HeuristicResult["facts"] = [];138 const pair = (kind: HeuristicResult["facts"][number]["kind"], re: RegExp): void => {139 const b = [...before.matchAll(re)].map((m) => m[0]);140 const a = [...after.matchAll(re)].map((m) => m[0]);141 if (!b.length && !a.length) return;142 const bs = new Set(b);143 const as = new Set(a);144 const gone = b.filter((x) => !as.has(x));145 const fresh = a.filter((x) => !bs.has(x));146 if (!gone.length && !fresh.length) return;147 const n = Math.max(gone.length, fresh.length);148 for (let i = 0; i < Math.min(n, 6); i++) facts.push({ kind, before: gone[i], after: fresh[i] });149 };150 pair("price", MONEY_RE);151 pair("percent", PERCENT_RE);152 pair("version", VERSION_RE);153 pair("date", DATE_RE);154 return facts;155}156157export function evaluateChange(diff: DiffResult, context: { sensorType: string; url: string; sourceCategories: string[]; title?: string | null }): HeuristicResult {158 const reasons: string[] = [];159 let changedLines: string[] = [];160 let beforeText = "";161 let afterText = "";162 let noiseRatio = 0;163 let magnitude = 0;164165 if (diff.kind === "text") {166 const all = [...diff.added, ...diff.removed, ...diff.modified.map((m) => m.after), ...diff.modified.map((m) => m.before)];167 const noisy = all.filter(isNoiseLine);168 noiseRatio = all.length ? noisy.length / all.length : 1;169 changedLines = all.filter((l) => !isNoiseLine(l));170 beforeText = [...diff.removed, ...diff.modified.map((m) => m.before)].join("\n");171 afterText = [...diff.added, ...diff.modified.map((m) => m.after)].join("\n");172 const churn = 1 - diff.stats.unchangedRatio;173 magnitude = Math.min(100, Math.round(100 * Math.min(1, churn * 2 + changedLines.length / 60)));174 if (!changedLines.length) reasons.push("all changed lines matched noise rules");175 } else if (diff.kind === "json") {176 const meaningful = diff.changes.filter((c) => !/(timestamp|updated_at|generated|nonce|etag|request_id|_id$|token|cache)/i.test(c.path));177 noiseRatio = diff.changes.length ? 1 - meaningful.length / diff.changes.length : 1;178 changedLines = meaningful.map((c) => `${c.path}: ${JSON.stringify(c.before)} → ${JSON.stringify(c.after)}`);179 beforeText = meaningful.map((c) => `${c.path}: ${JSON.stringify(c.before ?? "")}`).join("\n");180 afterText = meaningful.map((c) => `${c.path}: ${JSON.stringify(c.after ?? "")}`).join("\n");181 magnitude = Math.min(100, meaningful.length * 12);182 } else {183 const items = [...diff.added, ...diff.modified.map((m) => m.after)];184 changedLines = items.map((i) => [i.title, i.summary, i.url].filter(Boolean).join(" — ")).filter((s) => s.length);185 changedLines.push(...diff.removed.map((i) => `removed: ${String(i.title ?? i.url ?? i.key)}`));186 noiseRatio = 0;187 afterText = changedLines.join("\n");188 magnitude = Math.min(100, 20 + diff.added.length * 15 + diff.removed.length * 10 + diff.modified.length * 5);189 if (diff.added.length) reasons.push(`${diff.added.length} new item(s)`);190 if (diff.removed.length) reasons.push(`${diff.removed.length} removed item(s)`);191 }192193 const corpus = (context.title ? context.title + "\n" : "") + changedLines.join("\n");194 const scores = new Map<string, number>();195 const keywords = new Set<string>();196 for (const rule of TYPE_RULES) {197 const matches = corpus.match(new RegExp(rule.re.source, rule.re.flags.includes("g") ? rule.re.flags : rule.re.flags + "g"));198 if (!matches?.length) continue;199 const s = rule.weight * Math.min(3, matches.length);200 scores.set(rule.type, (scores.get(rule.type) ?? 0) + s);201 for (const m of matches.slice(0, 3)) keywords.add(m.toLowerCase());202 }203204 // Sensor-type priors: a new item in a status feed is an incident, in a release feed a release…205 const st = context.sensorType;206 if (diff.kind === "list" && diff.added.length) {207 if (st === "STATUSPAGE") scores.set("incident", (scores.get("incident") ?? 0) + 2);208 if (st === "GITHUB_RELEASE") scores.set("repository_release", (scores.get("repository_release") ?? 0) + 2);209 if (st === "SITEMAP") scores.set("page_created", (scores.get("page_created") ?? 0) + 1.5);210 if (st === "RSS" || st === "ATOM") scores.set("announcement", (scores.get("announcement") ?? 0) + 1);211 }212 if (diff.kind === "list" && diff.removed.length && st === "SITEMAP") scores.set("page_removed", (scores.get("page_removed") ?? 0) + 1.5);213 // Connector-class priors (2026-09-08): the shape of the items / the sensor type tells the class.214 const bump = (t: string, v: number): void => void scores.set(t, (scores.get(t) ?? 0) + v);215 if (diff.kind === "json") {216 if (st === "TLS") bump("certificate_change", 2.5);217 if (st === "DNS" && !/rdap/i.test(context.url)) bump("dns_change", 2.5);218 if (/rdap\./i.test(context.url) || /\/domain\//.test(context.url)) bump("domain_registration_change", 2.5);219 if (st === "HTTP_HEADERS") bump("infrastructure_change", 2);220 if (diff.changes.some((c) => /^headers\.(strict-transport-security|content-security-policy|x-frame-options|permissions-policy)/i.test(c.path))) bump("policy_change", 1.2);221 }222 if (diff.kind === "text" && /\/robots\.txt$|\/llms\.txt$|\/ai\.txt$/i.test(context.url)) {223 bump("crawler_policy_change", 3);224 if (/gptbot|claudebot|anthropic|ccbot|google-extended|perplexitybot|bytespider|applebot-extended|meta-externalagent|amazonbot|cohere|ai2bot/i.test(corpus)) {225 bump("crawler_policy_change", 2);226 reasons.push("AI crawler directive changed");227 }228 }229 if (diff.kind === "text" && /security\.txt$/i.test(context.url)) bump("policy_change", 1.5);230 if (diff.kind === "list") {231 const fresh = [...diff.added, ...diff.modified.map((m) => m.after)];232 const has = (f: string): boolean => fresh.some((i) => i && typeof i === "object" && f in i);233 if (has("form")) {234 bump("financial_filing", 2.5);235 const forms = fresh.map((i) => String((i as { form?: string }).form ?? ""));236 const items8k = fresh.map((i) => String((i as { items?: string }).items ?? "")).join(",");237 if (forms.some((f) => /^(10-K|10-Q|20-F|40-F)/.test(f))) bump("earnings", 1.5);238 // Decoded 8-K items are more specific than the generic filing prior — let them win.239 if (/\b2\.02\b/.test(items8k)) bump("earnings", 4);240 if (/\b5\.02\b/.test(items8k)) bump("leadership_change", 4);241 if (/\b1\.05\b/.test(items8k)) bump("breach", 4.5);242 if (/\b2\.01\b/.test(items8k)) bump("acquisition", 4);243 if (/\b1\.03\b/.test(items8k)) bump("financial_filing", 1);244 if (forms.some((f) => /^(S-1|F-1|424B4)/.test(f))) bump("funding", 2);245 if (forms.some((f) => /^(SC 13D|SC 13G|SC TO)/.test(f))) bump("acquisition", 1.5);246 if (forms.some((f) => /^(DEF 14A|DEFA14A|PRE 14A)/.test(f))) bump("regulatory_filing", 1);247 }248 if (has("version") && (has("prerelease") || has("digest"))) bump("software_release", 2.5);249 if (has("fingerprint")) {250 bump("api_change", 3);251 if (fresh.some((i) => (i as { deprecated?: boolean }).deprecated)) reasons.push("operation deprecated");252 if (diff.removed.length) reasons.push(`${diff.removed.length} operation(s) removed`);253 }254 if (has("row")) bump(context.sourceCategories.some((c) => ["finance", "statistics", "central-bank", "open-data"].includes(c)) ? "economic_release" : "dataset_release", 2);255 if (has("kind") && fresh.some((i) => (i as { kind?: string }).kind === "incident")) bump("incident", 2);256 if (has("kind") && fresh.some((i) => (i as { kind?: string }).kind === "maintenance")) bump("maintenance", 2);257 // CISA KEV records (cveID/dateAdded/knownRansomwareCampaignUse) = confirmed active exploitation.258 if (has("cveID") || fresh.some((i) => /known_exploited|kev/i.test(String((i as { url?: string }).url ?? "")))) {259 bump("active_exploitation", 5);260 if (fresh.some((i) => /known/i.test(String((i as { knownRansomwareCampaignUse?: string }).knownRansomwareCampaignUse ?? "")))) bump("ransomware", 2);261 }262 // NVD / GHSA / OSV records: the record class beats free-text words such as "compromised" inside CVE descriptions.263 if (fresh.some((i) => /^CVE-\d{4}-\d{4,}$/i.test(String(i.key ?? "")) || /^GHSA-/i.test(String(i.key ?? "")) || /^EUVD-/i.test(String(i.key ?? "")))) {264 bump("vulnerability", 6);265 for (const t of ["breach", "credential_leak", "malware_campaign", "ransomware"]) scores.set(t, (scores.get(t) ?? 0) * 0.4);266 }267 if (has("form") && fresh.some((i) => /^4$/.test(String((i as { form?: string }).form ?? "")))) bump("insider_transaction", 3);268 if (has("form") && fresh.some((i) => /^(8-K)/.test(String((i as { form?: string }).form ?? "")) && /\b1\.01\b/.test(String((i as { items?: string }).items ?? "")))) bump("acquisition", 1);269 // Clinical-trial records with a status field270 if (fresh.some((i) => /nct\d{8}/i.test(String(i.key ?? "")))) {271 bump("clinical_trial", 3);272 if (fresh.some((i) => /terminated|withdrawn|suspended/i.test(JSON.stringify(i))) || diff.modified.some((m) => m.fields.some((f) => /status/i.test(f)) && /terminated|withdrawn|suspended/i.test(JSON.stringify(m.after)))) bump("clinical_trial", 2);273 }274 // Federal-register style records275 if (fresh.some((i) => /\b(RULE|PRORULE|NOTICE|PRESDOCU)\b/.test(String((i as { type?: string }).type ?? "")))) bump("regulatory_filing", 2);276 }277 // Sports sources: keep results/transactions in their own low-severity classes instead of "announcement".278 if (context.sourceCategories.includes("sports") && diff.kind === "list" && diff.added.length) {279 bump("sports_result", 0.8);280 bump("sports_transaction", 0.6);281 }282 if (/recalls?-rappels|enforcement\.json|\/recall/i.test(context.url)) bump(/device/i.test(context.url) ? "device_recall" : "recall", 2.5);283 if (/clinicaltrials\.gov/i.test(context.url)) bump("clinical_trial", 2);284 if (/federalregister|gazette|legislation\.gov|eur-lex|legisinfo|parl\.ca|congress\.gov|assnat|boe\.es|legifrance/i.test(context.url)) bump("legislation", 1.5);285 if (/sanction|ofac/i.test(context.url)) bump("sanction", 2);286 if (/tender|procurement|canadabuys|ted\.europa|find-tender|seao/i.test(context.url)) bump("procurement", 2.5);287 if (/earthquake|alerts?\/active|weather\.gov\/alerts|battleboard|warnings/i.test(context.url)) bump("emergency_notice", 1.5);288 if (/known_exploited|kev/i.test(context.url)) bump("active_exploitation", 2);289 if (/pricing|price/i.test(context.url)) scores.set("pricing_change", (scores.get("pricing_change") ?? 0) + 1.5);290 if (/terms|tos\b/i.test(context.url)) scores.set("terms_change", (scores.get("terms_change") ?? 0) + 1.5);291 if (/polic/i.test(context.url)) scores.set("policy_change", (scores.get("policy_change") ?? 0) + 1.2);292 if (/docs?\.|\/docs?\/|documentation|reference/i.test(context.url)) scores.set("documentation_change", (scores.get("documentation_change") ?? 0) + 0.8);293 if (/status\./i.test(context.url)) scores.set("incident", (scores.get("incident") ?? 0) + 0.8);294 if (/changelog|release/i.test(context.url)) scores.set("software_release", (scores.get("software_release") ?? 0) + 0.8);295 if (/security|advisor/i.test(context.url)) scores.set("security_advisory", (scores.get("security_advisory") ?? 0) + 1);296297 const facts = extractFacts(beforeText, afterText);298 if (facts.some((f) => f.kind === "price" && f.before && f.after)) {299 // A changed money amount is a pricing signal only in a pricing context; statistical releases, budgets and300 // earnings are full of dollar figures that are not prices.301 const pricingContext = /pricing|price|plans?\b|billing|subscri|tarif/i.test(context.url) || /\b(per month|per seat|per user|\/mo\b|per (million|1k|1m) tokens|free tier|rate card|starter plan|pro plan|enterprise plan)\b/i.test(corpus);302 const statistical = context.sourceCategories.some((c) => ["statistics", "government", "finance", "central-bank", "open-data"].includes(c));303 if (pricingContext) {304 scores.set("pricing_change", (scores.get("pricing_change") ?? 0) + 3);305 reasons.push("price value changed");306 } else if (!statistical) {307 scores.set("pricing_change", (scores.get("pricing_change") ?? 0) + 1.2);308 reasons.push("money amount changed (no pricing context)");309 }310 }311 if (facts.some((f) => f.kind === "version" && f.before && f.after)) {312 scores.set("software_release", (scores.get("software_release") ?? 0) + 1.5);313 reasons.push("version number changed");314 }315316 let eventType = "content_change";317 let best = 0;318 for (const [t, s] of scores) {319 // prefer the more specific / severe type when tied320 const sev = eventTypeSpec(t).severity / 100;321 const v = s + sev * 0.5;322 if (v > best) {323 best = v;324 eventType = t;325 }326 }327 if (!changedLines.length) eventType = "unknown";328 if (eventType === "unknown" && diff.kind === "list" && (diff.added.length || diff.removed.length)) eventType = diff.added.length ? "announcement" : "page_removed";329330 // Signal: meaningful lines, type confidence, magnitude331 const contentSignal = Math.min(1, changedLines.length / 3) * (1 - noiseRatio * 0.7);332 const typeSignal = Math.min(1, best / 3);333 let signal = Math.max(0, Math.min(1, 0.55 * contentSignal + 0.35 * typeSignal + 0.1 * (magnitude / 100)));334 if (!changedLines.length) signal = 0;335 if (changedLines.length === 1 && changedLines[0]!.length < 12 && !facts.length) signal = Math.min(signal, 0.15);336 if (!(eventType in EVENT_TYPES)) eventType = "unknown";337338 return {339 signal,340 noiseRatio,341 eventType,342 magnitude,343 keywords: [...keywords].slice(0, 12),344 reasons,345 facts,346 };347}348349/** "https://x.com/sustainability/performance-reports/news/detail/index_4751.html" → "sustainability / performance reports / news / detail / index 4751" */350export function humanizeUrl(u: string): string {351 try {352 const url = new URL(u);353 const parts = url.pathname354 .split("/")355 .filter(Boolean)356 .map((p) => decodeURIComponent(p).replace(/\.(html?|php|aspx?|jsp|pdf|xml|json)$/i, "").replace(/[-_]+/g, " ").trim())357 .filter((p) => p && !/^(index|default|home)$/i.test(p));358 const label = parts.slice(-4).join(" / ");359 return label ? `${label} (${url.hostname.replace(/^www\./, "")})` : url.hostname.replace(/^www\./, "");360 } catch {361 return u;362 }363}364365function itemLabel(i: { key: string; [k: string]: unknown }): string {366 const t = i.title;367 if (typeof t === "string" && t.trim() && !/^https?:\/\//i.test(t.trim())) return t.trim();368 const u = i.url;369 if (typeof u === "string" && /^https?:\/\//i.test(u)) return humanizeUrl(u);370 if (typeof t === "string" && t.trim()) return humanizeUrl(t.trim());371 return String(i.key);372}373374/** Deterministic fallback title/summary when no LLM is used. */375export function describeChange(h: HeuristicResult, diff: DiffResult, ctx: { sourceName: string; url: string; sensorName: string }): { title: string; summary: string } {376 const spec = eventTypeSpec(h.eventType);377 const price = h.eventType === "pricing_change" ? h.facts.find((f) => f.kind === "price" && f.before && f.after) : undefined;378 if (price) return { title: `${ctx.sourceName}: price changed ${price.before} → ${price.after}`, summary: `A price on ${ctx.url} changed from ${price.before} to ${price.after}.` };379 if (diff.kind === "list") {380 const first = diff.added[0];381 if (first && diff.added.length === 1) {382 const t = itemLabel(first);383 return { title: `${ctx.sourceName}: ${t}`.slice(0, 180), summary: String(first.summary ?? `New item published in ${ctx.sensorName}.`).slice(0, 600) };384 }385 if (diff.added.length > 1) {386 const labels = diff.added.slice(0, 3).map(itemLabel);387 const noun = /cve|vulnerab/i.test(ctx.sensorName + " " + ctx.url) ? "CVEs" : /release|tag/i.test(ctx.sensorName) ? "releases" : /filing|edgar/i.test(ctx.sensorName + " " + ctx.url) ? "filings" : /sitemap/i.test(ctx.sensorName) ? "pages" : /status|incident/i.test(ctx.sensorName + " " + ctx.url) ? "incidents" : "items";388 const head = `${ctx.sourceName}: ${diff.added.length} new ${noun} in ${ctx.sensorName}`;389 const title = `${head} — ${labels.join(" · ")}`.slice(0, 180);390 return { title: title.length >= 180 ? head.slice(0, 180) : title, summary: diff.added.slice(0, 6).map((i) => `• ${itemLabel(i)}`).join("\n") };391 }392 if (diff.removed.length) return { title: `${ctx.sourceName}: ${diff.removed.length} item(s) removed from ${ctx.sensorName}`, summary: diff.removed.slice(0, 5).map((i) => `• ${itemLabel(i)}`).join("\n") };393 }394 if (diff.kind === "json") {395 const c = diff.changes[0];396 if (c) return { title: `${ctx.sourceName}: ${spec.label.toLowerCase()} (${c.path})`, summary: `${diff.changes.length} field(s) changed on ${ctx.url}. First: ${c.path} ${JSON.stringify(c.before)} → ${JSON.stringify(c.after)}`.slice(0, 600) };397 }398 if (diff.kind === "text") {399 const sample = (diff.modified[0]?.after ?? diff.added[0] ?? diff.removed[0] ?? "").slice(0, 140);400 return {401 title: `${ctx.sourceName}: ${spec.label.toLowerCase()} on ${ctx.sensorName}`.slice(0, 180),402 summary: `${diff.stats.added} line(s) added, ${diff.stats.removed} removed, ${diff.stats.modified} modified on ${ctx.url}.${sample ? ` Example: “${sample}”` : ""}`,403 };404 }405 return { title: `${ctx.sourceName}: ${spec.label.toLowerCase()}`, summary: `Change detected on ${ctx.url}.` };406}407