TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import type { DiffResult } from "./diff";2import type { ChangeClass } from "./taxonomy";34/**5 * Semantic diff (spec §21–22, §55). Two jobs:6 * 1. `classifyChange` — decide what KIND of change this is (cosmetic / navigation / timestamp /7 * advertisement / boilerplate = noise; meaningful / pricing / policy / product / personnel = signal)8 * so that footer-year bumps and "3 min read" counters never become events.9 * 2. `extractFieldChanges` — turn a raw diff into field-level "before → after" pairs (price $20 → $25,10 * context window 128k → 256k, status beta → GA, CEO A → B) that the UI shows as WHAT changed.11 * Everything here is deterministic and dependency-free (browser-safe).12 */1314export interface FieldChange {15 label: string;16 kind: "price" | "percent" | "number" | "date" | "version" | "status" | "text";17 before: string | null;18 after: string | null;19 deltaPct?: number | null;20}2122export interface SemanticResult {23 class: ChangeClass;24 /** 0–1 confidence in the class */25 confidence: number;26 reasons: string[];27 contentLines: number;28 noiseLines: number;29 /** share of changed lines that are noise */30 noiseRatio: number;31 fieldChanges: FieldChange[];32}3334// ---- Line classifiers -----------------------------------------------------------------------3536const TS_PREFIX = "(?:(?:updated|last updated|published|posted|modified|generated|as of|effective|revised|date)\\s*:?\\s*)?";37const TS_ISO = "\\d{4}-\\d{2}-\\d{2}(?:[t ]\\d{2}:\\d{2}(?::\\d{2})?(?:\\.\\d+)?(?:z|[+-]\\d{2}:?\\d{2})?)?";38const TS_MONTH = "(?:(?:mon|tue|wed|thu|fri|sat|sun)[a-z]*,?\\s+)?(?:\\d{1,2}\\s+)?(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\\.?\\s+\\d{1,2}(?:st|nd|rd|th)?,?\\s+\\d{4}(?:\\s+\\d{1,2}:\\d{2}(?::\\d{2})?\\s*(?:am|pm|utc|gmt|est|edt|pst|pdt|cet|cest)?)?";39const TS_CLOCK = "\\d{1,2}:\\d{2}(?::\\d{2})?\\s*(?:am|pm|utc|gmt)?";40const TS_REL = "\\d+\\s+(?:seconds?|minutes?|mins?|hours?|days?|weeks?|months?)\\s+ago";41const TS_WORD = "(?:today|yesterday|just now)";42const RE_TIMESTAMP_ONLY = new RegExp(`^\\s*${TS_PREFIX}(?:${TS_ISO}|${TS_MONTH}|${TS_CLOCK}|${TS_REL}|${TS_WORD})\\s*$`, "i");43const RE_READ_TIME = /^\s*\d+\s*(?:min|minute)s?\s*read\s*$/i;44const RE_COUNTER = /^\s*[\d.,]+\s*[kKmM]?\s*(?:views?|likes?|shares?|comments?|followers?|stars?|forks?|downloads?|reactions?|votes?|points?|replies)?\s*$/i;45const RE_COPYRIGHT = /©|\(c\)\s*\d{4}|\bcopyright\b|\ball rights reserved\b|\btous droits réservés\b/i;46const RE_YEAR_ONLY = /^\s*(?:19|20)\d{2}(?:\s*[-–]\s*(?:19|20)\d{2})?\s*$/;47const RE_COOKIE = /\bcookies?\b|\bconsent\b|privacy (?:preferences|settings|choices)|accept all|reject all|manage preferences|gdpr|ccpa|do not sell/i;48const RE_ADVERT = /\b(?:sponsored|advertisement|advertising|promoted|promo code|coupon|limited time offer|black friday|cyber monday|save \d+%|newsletter|subscribe now|sign up for our|join our mailing|get the app|download the app|utm_|doubleclick|adsbygoogle|taboola|outbrain)\b/i;49const RE_NAV_WORD = /^(?:home|menu|search|login|log in|sign in|sign up|register|subscribe|share|print|email|back to top|skip to (?:main )?content|close|next|previous|prev|read more|learn more|more|see all|view all|show more|show less|open menu|close menu|toggle navigation|about|about us|contact|contact us|careers|blog|news|pricing|docs|documentation|support|help|faq|terms|privacy|legal|sitemap|english|français|español|deutsch|en|fr|es|de|follow us|twitter|x|facebook|linkedin|youtube|instagram|github|rss|download|products?|solutions?|resources?|company|partners?|customers?|developers?|community|events?|press|investors?|status|changelog|api|login\/register|accessibility|cookie settings|feedback)$/i;50const RE_SOCIAL = /\b(?:tweet|share on (?:x|twitter|facebook|linkedin)|copy link|whatsapp|telegram)\b/i;51const RE_BREADCRUMB = /^(?:[\w .'&-]{1,30}\s*[›>/»]\s*){1,6}[\w .'&-]{1,40}$/;5253type LineClass = "timestamp" | "navigation" | "advertisement" | "boilerplate" | "content";5455export function classifyLine(line: string): LineClass {56 const l = line.trim();57 if (!l || l.length < 3) return "boilerplate";58 if (RE_YEAR_ONLY.test(l) || RE_COPYRIGHT.test(l) || RE_COOKIE.test(l)) return "boilerplate";59 if (RE_TIMESTAMP_ONLY.test(l) || RE_READ_TIME.test(l) || RE_COUNTER.test(l)) return "timestamp";60 if (RE_ADVERT.test(l) && l.length < 160) return "advertisement";61 if (RE_NAV_WORD.test(l) || RE_SOCIAL.test(l) || (l.length < 80 && RE_BREADCRUMB.test(l) && !/\d/.test(l))) return "navigation";62 // very short label-like fragments in a nav-like burst are handled at the diff level63 return "content";64}6566/** True when two lines differ only by whitespace, punctuation, case or quotes. */67export function isCosmeticPair(before: string, after: string): boolean {68 const norm = (s: string): string => s.toLowerCase().replace(/[\s ]+/g, " ").replace(/[“”"'`´‘’«»]/g, "").replace(/[.,;:!?…\-–—()[\]{}]/g, "").trim();69 return norm(before) === norm(after);70}7172/** True when the only difference between two lines is a timestamp / counter / relative time token. */73export function isTimestampPair(before: string, after: string): boolean {74 const strip = (s: string): string => s.replace(/\d{4}-\d{2}-\d{2}(?:[t ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?z?)?/gi, "␣").replace(/\b\d{1,2}:\d{2}(?::\d{2})?\s*(?:am|pm|utc|gmt)?/gi, "␣").replace(/\b\d+\s+(?:seconds?|minutes?|mins?|hours?|days?|weeks?)\s+ago/gi, "␣").replace(/\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4}/gi, "␣").replace(/\b\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{4}/gi, "␣").replace(/\b[\d.,]+\s*[km]?\s*(?:views?|likes?|shares?|comments?|followers?|stars?|forks?|downloads?)\b/gi, "␣");75 const a = strip(before);76 const b = strip(after);77 return a === b && a !== before;78}7980// ---- Field extraction ---------------------------------------------------------------------------8182const MONEY = /(?:(?:US|CA|AU|NZ|HK|SG)?\$|€|£|¥|₹|CHF\s?|USD\s?|CAD\s?|EUR\s?|GBP\s?)\s?\d[\d,]*(?:\.\d+)?(?:\s?(?:k|m|bn?|million|billion|trillion))?(?:\s?(?:\/|per)\s?(?:1k|1m|million|m|k)?\s?(?:tokens?|month|mo|year|yr|seat|user|hour|hr|gb|tb|request|call|image|minute|min))?/gi;83const PERCENT = /-?\d+(?:\.\d+)?\s?%/g;84const VERSION = /\bv?\d+\.\d+(?:\.\d+){0,2}(?:[-.](?:alpha|beta|rc|preview|dev|nightly|lts)\.?\d*)?\b/gi;85const DATE = /\b(?:\d{4}-\d{2}-\d{2}|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{4})\b/gi;86const NUMBER = /\b\d[\d,]*(?:\.\d+)?\s?(?:k|m|b|bn|million|billion|trillion|gb|tb|mb|ms|s|tokens?|rpm|tpm|req\/s|qps|users?|seats?|cores?|vcpus?|nodes?|regions?|countries|employees|people|jobs|positions)?\b/gi;87const STATUS_WORDS = /\b(?:beta|alpha|preview|public preview|private preview|early access|general availability|generally available|ga|stable|production|deprecated|sunset|retired|discontinued|end of life|eol|coming soon|waitlist|available|unavailable|limited|paused|suspended|resolved|investigating|identified|monitoring|degraded|operational|partial outage|major outage|under maintenance|active|inactive|open|closed|approved|rejected|pending|recalled|withdrawn|terminated|completed|recruiting|not yet recruiting|enrolling)\b/gi;8889function toNumber(s: string): number | null {90 const m = s.replace(/,/g, "").match(/-?\d+(?:\.\d+)?/);91 if (!m) return null;92 let n = Number(m[0]);93 const l = s.toLowerCase();94 if (/\b(?:bn|billion)\b|(?<=\d)b\b/.test(l)) n *= 1e9;95 else if (/\bmillion\b|(?<=\d)\s?m\b/.test(l) && !/\bms\b|\bmb\b/.test(l)) n *= 1e6;96 else if (/(?<=\d)\s?k\b/.test(l)) n *= 1e3;97 return Number.isFinite(n) ? n : null;98}99100function deltaPct(before: string | null, after: string | null): number | null {101 if (!before || !after) return null;102 const a = toNumber(before);103 const b = toNumber(after);104 if (a === null || b === null || a === 0) return null;105 return Math.round(((b - a) / Math.abs(a)) * 1000) / 10;106}107108/** Words immediately before a token → a human label ("Input tokens", "Context window"). */109function labelFor(line: string, token: string, fallback: string): string {110 const i = line.indexOf(token);111 const head = (i > 0 ? line.slice(0, i) : line.slice(i + token.length)).replace(/[:\-–—|]+\s*$/, "").trim();112 const words = head.split(/\s+/).filter(Boolean);113 const label = words.slice(-6).join(" ").replace(/^[^\p{L}\p{N}]+/u, "");114 return (label.length >= 3 ? label : fallback).slice(0, 60);115}116117function pairTokens(before: string, after: string, re: RegExp, kind: FieldChange["kind"], out: FieldChange[], seen: Set<string>): void {118 const b = [...before.matchAll(re)].map((m) => m[0].trim());119 const a = [...after.matchAll(re)].map((m) => m[0].trim());120 if (!b.length && !a.length) return;121 const bs = new Set(b);122 const as = new Set(a);123 const gone = b.filter((x) => !as.has(x));124 const fresh = a.filter((x) => !bs.has(x));125 if (!gone.length && !fresh.length) return;126 const n = Math.max(gone.length, fresh.length);127 for (let i = 0; i < Math.min(n, 4); i++) {128 const bef = gone[i] ?? null;129 const aft = fresh[i] ?? null;130 const key = `${kind}|${bef}|${aft}`;131 if (seen.has(key)) continue;132 seen.add(key);133 const label = labelFor(aft ? after : before, aft ?? bef ?? "", kind === "price" ? "Price" : kind === "percent" ? "Rate" : kind === "version" ? "Version" : kind === "date" ? "Date" : kind === "status" ? "Status" : "Value");134 out.push({ label, kind, before: bef, after: aft, deltaPct: kind === "price" || kind === "number" || kind === "percent" ? deltaPct(bef, aft) : null });135 }136}137138export function extractFieldChanges(diff: DiffResult, max = 12): FieldChange[] {139 const out: FieldChange[] = [];140 const seen = new Set<string>();141 if (diff.kind === "text") {142 // Pair removed/added lines that share a "Label:" prefix (the line differ only pairs similar strings,143 // so "Status: beta" → "Status: general availability" arrives as one removal + one addition).144 const labelled = (lines: string[]): Map<string, string> => {145 const m = new Map<string, string>();146 for (const l of lines) {147 const x = l.match(/^\s*([^:]{2,40}):\s*(.{1,120})$/);148 if (x && !m.has(x[1]!.trim().toLowerCase())) m.set(x[1]!.trim().toLowerCase(), l.trim());149 }150 return m;151 };152 const rem = labelled(diff.removed);153 const add = labelled(diff.added);154 const pairs: { before: string; after: string }[] = [...diff.modified];155 for (const [k, b] of rem) {156 const a = add.get(k);157 if (a && a !== b) pairs.push({ before: b, after: a });158 }159 for (const m of pairs) {160 if (isCosmeticPair(m.before, m.after) || isTimestampPair(m.before, m.after)) continue;161 pairTokens(m.before, m.after, MONEY, "price", out, seen);162 pairTokens(m.before, m.after, PERCENT, "percent", out, seen);163 pairTokens(m.before, m.after, VERSION, "version", out, seen);164 pairTokens(m.before, m.after, DATE, "date", out, seen);165 pairTokens(m.before, m.after, STATUS_WORDS, "status", out, seen);166 // plain numbers only when nothing more specific was found on this pair167 if (!out.some((f) => f.kind !== "text" && (m.before.includes(f.before ?? " ") || m.after.includes(f.after ?? " ")))) pairTokens(m.before, m.after, NUMBER, "number", out, seen);168 if (out.length >= max) break;169 }170 // Whole-line replacements that are short and look like a labelled value ("CEO: Jane Doe")171 if (out.length < max) {172 for (const m of pairs) {173 const lb = m.before.match(/^([^:]{2,40}):\s*(.{1,80})$/);174 const la = m.after.match(/^([^:]{2,40}):\s*(.{1,80})$/);175 if (lb && la && lb[1]!.trim().toLowerCase() === la[1]!.trim().toLowerCase() && lb[2] !== la[2]) {176 const key = `text|${lb[2]}|${la[2]}`;177 if (seen.has(key)) continue;178 seen.add(key);179 out.push({ label: lb[1]!.trim().slice(0, 60), kind: "text", before: lb[2]!.trim(), after: la[2]!.trim(), deltaPct: null });180 if (out.length >= max) break;181 }182 }183 }184 } else if (diff.kind === "json") {185 for (const c of diff.changes) {186 if (/(timestamp|updated_at|updatedAt|generated|nonce|etag|request_id|_id$|token|cache|expires|ttl|lastModified|last_modified)/i.test(c.path)) continue;187 const b = c.before === undefined ? null : typeof c.before === "string" ? c.before : JSON.stringify(c.before);188 const a = c.after === undefined ? null : typeof c.after === "string" ? c.after : JSON.stringify(c.after);189 if (b !== null && a !== null && b.length > 200 && a.length > 200) continue;190 const num = typeof c.before === "number" && typeof c.after === "number";191 const kind: FieldChange["kind"] = num ? "number" : b && a && MONEY.test(b + a) ? "price" : b && a && STATUS_WORDS.test(a) ? "status" : "text";192 MONEY.lastIndex = 0;193 STATUS_WORDS.lastIndex = 0;194 const label = c.path.replace(/^\$\.?/, "").split(".").slice(-2).join(" · ").replace(/\[\d+\]/g, "").replace(/[_-]+/g, " ") || "value";195 out.push({ label: label.slice(0, 60), kind, before: b, after: a, deltaPct: num ? deltaPct(String(c.before), String(c.after)) : null });196 if (out.length >= max) break;197 }198 } else {199 for (const m of diff.modified) {200 for (const f of m.fields) {201 const b = m.before[f];202 const a = m.after[f];203 const bs = b === undefined || b === null ? null : typeof b === "string" ? b : JSON.stringify(b);204 const as = a === undefined || a === null ? null : typeof a === "string" ? a : JSON.stringify(a);205 if (bs === as) continue;206 const label = `${String(m.after.title ?? m.after.name ?? m.key).slice(0, 40)} · ${f}`;207 const kind: FieldChange["kind"] = typeof b === "number" && typeof a === "number" ? "number" : as && STATUS_WORDS.test(as) ? "status" : "text";208 STATUS_WORDS.lastIndex = 0;209 out.push({ label, kind, before: bs && bs.length > 160 ? bs.slice(0, 157) + "…" : bs, after: as && as.length > 160 ? as.slice(0, 157) + "…" : as, deltaPct: kind === "number" ? deltaPct(String(b), String(a)) : null });210 if (out.length >= max) return out;211 }212 }213 }214 return out.slice(0, max);215}216217// ---- Change classification ------------------------------------------------------------------------218219const PRICING_WORDS = /\b(price|pricing|per (?:million|1k|1m) tokens|\$\s?\d|€\s?\d|£\s?\d|usd|cad|eur|per month|per seat|per user|\/mo\b|billing|discount|free tier|rate card|fee|fees|subscription|plan)\b/i;220const POLICY_WORDS = /\b(policy|policies|terms of (?:service|use)|terms and conditions|acceptable use|privacy policy|guidelines|code of conduct|license|licence|eula|agreement|compliance|gdpr|data processing|retention|prohibited|must not|may not|shall)\b/i;221const PERSONNEL_WORDS = /\b(ceo|cfo|cto|coo|cio|ciso|chief \w+ officer|president|chair(?:man|woman|person)?|director|vice president|vp\b|head of|appointed|appointment|steps? down|resign|joins|joined|hired|departure|leadership|board member|founder|general manager|managing director)\b/i;222const PRODUCT_WORDS = /\b(launch|introducing|now available|new model|model|version|release|feature|api|sdk|endpoint|context window|parameters|benchmark|availability|preview|beta|general availability|deprecat|sunset|discontinu|end of life|region|integration|update)\b/i;223224export function classifyChange(diff: DiffResult, ctx: { url: string; sensorType: string; title?: string | null }): SemanticResult {225 const reasons: string[] = [];226 let content = 0;227 const noise: Record<Exclude<LineClass, "content">, number> = { timestamp: 0, navigation: 0, advertisement: 0, boilerplate: 0 };228 let cosmetic = 0;229 const contentText: string[] = [];230231 if (diff.kind === "text") {232 for (const m of diff.modified) {233 if (isCosmeticPair(m.before, m.after)) {234 cosmetic++;235 continue;236 }237 if (isTimestampPair(m.before, m.after)) {238 noise.timestamp++;239 continue;240 }241 const c = classifyLine(m.after);242 if (c === "content") {243 content++;244 contentText.push(m.before, m.after);245 } else noise[c]++;246 }247 for (const l of [...diff.added, ...diff.removed]) {248 const c = classifyLine(l);249 if (c === "content") {250 content++;251 contentText.push(l);252 } else noise[c]++;253 }254 // A burst of many short lines with no sentence = navigation/template churn (site redesign).255 const shortFragments = [...diff.added, ...diff.removed].filter((l) => l.trim().length < 28 && !/[.!?]/.test(l)).length;256 if (shortFragments >= 12 && shortFragments / Math.max(1, diff.added.length + diff.removed.length) > 0.6) {257 reasons.push("burst of short label-like fragments (template/navigation churn)");258 noise.navigation += Math.round(shortFragments * 0.5);259 content = Math.max(0, content - Math.round(shortFragments * 0.5));260 }261 } else if (diff.kind === "json") {262 for (const c of diff.changes) {263 if (/(timestamp|updated_at|updatedAt|generated|nonce|etag|request_id|_id$|token|cache|expires|ttl|lastModified|last_modified|servedAt|fetchedAt|\bdate$)/i.test(c.path)) noise.timestamp++;264 else {265 content++;266 contentText.push(`${c.path}: ${JSON.stringify(c.before ?? "")} → ${JSON.stringify(c.after ?? "")}`);267 }268 }269 } else {270 content = diff.added.length + diff.removed.length + diff.modified.length;271 for (const i of [...diff.added, ...diff.modified.map((m) => m.after)]) contentText.push(String(i.title ?? ""), String(i.summary ?? ""));272 for (const i of diff.removed) contentText.push(String(i.title ?? i.url ?? i.key));273 }274275 const noiseTotal = noise.timestamp + noise.navigation + noise.advertisement + noise.boilerplate + cosmetic;276 const total = content + noiseTotal;277 const noiseRatio = total ? noiseTotal / total : 1;278 const fieldChanges = extractFieldChanges(diff);279 const corpus = `${ctx.title ?? ""}\n${contentText.join("\n")}`.slice(0, 20_000);280281 let cls: ChangeClass;282 let confidence: number;283 if (total === 0) {284 cls = "cosmetic";285 confidence = 0.9;286 reasons.push("no visible change after canonicalization");287 } else if (content === 0 || (content <= 1 && noiseRatio >= 0.75 && !fieldChanges.some((f) => f.kind === "price" || f.kind === "status"))) {288 const top = (Object.entries(noise) as [Exclude<LineClass, "content">, number][]).sort((a, b) => b[1] - a[1])[0];289 cls = cosmetic >= (top?.[1] ?? 0) ? "cosmetic" : (top?.[0] ?? "boilerplate");290 confidence = 0.85;291 reasons.push(`all ${total} changed line(s) are ${cls === "cosmetic" ? "punctuation/whitespace/case" : cls} noise`);292 } else {293 // signal — pick the most specific sub-class294 const money = fieldChanges.filter((f) => f.kind === "price").length;295 const pricingHits = (corpus.match(new RegExp(PRICING_WORDS.source, "gi")) ?? []).length;296 const policyHits = (corpus.match(new RegExp(POLICY_WORDS.source, "gi")) ?? []).length;297 const personnelHits = (corpus.match(new RegExp(PERSONNEL_WORDS.source, "gi")) ?? []).length;298 const productHits = (corpus.match(new RegExp(PRODUCT_WORDS.source, "gi")) ?? []).length;299 const urlPricing = /pricing|price|plans?\b|billing/i.test(ctx.url);300 const urlPolicy = /terms|tos\b|polic|privacy|legal|license|acceptable-use|eula/i.test(ctx.url);301 const scores: [ChangeClass, number][] = [302 ["pricing", money * 3 + pricingHits + (urlPricing ? 3 : 0)],303 ["policy", policyHits * 1.2 + (urlPolicy ? 3 : 0)],304 ["personnel", personnelHits * 1.5],305 ["product", productHits * 0.8 + fieldChanges.filter((f) => f.kind === "version" || f.kind === "status").length * 2],306 ];307 scores.sort((a, b) => b[1] - a[1]);308 const [best, bestScore] = scores[0]!;309 if (bestScore >= 3) {310 cls = best;311 confidence = Math.min(0.95, 0.55 + bestScore / 20);312 reasons.push(`${best} vocabulary dominates (${bestScore.toFixed(1)})`);313 } else {314 cls = "meaningful";315 confidence = Math.min(0.9, 0.5 + content / 20) * (1 - noiseRatio * 0.5);316 }317 if (noiseRatio > 0.5) reasons.push(`${Math.round(noiseRatio * 100)}% of changed lines are template noise`);318 if (fieldChanges.length) reasons.push(`${fieldChanges.length} field-level change(s) extracted`);319 }320 return { class: cls, confidence: Math.round(confidence * 100) / 100, reasons, contentLines: content, noiseLines: noiseTotal, noiseRatio: Math.round(noiseRatio * 100) / 100, fieldChanges };321}322323/** Compact, human-readable "what changed" line built from field changes (used in titles/summaries). */324export function describeFieldChanges(f: FieldChange[], max = 3): string {325 return f326 .slice(0, max)327 .map((x) => `${x.label}: ${x.before ?? "∅"} → ${x.after ?? "∅"}${x.deltaPct !== null && x.deltaPct !== undefined ? ` (${x.deltaPct > 0 ? "+" : ""}${x.deltaPct}%)` : ""}`)328 .join(" · ");329}330