import { clamp01, type Evidenced, type ObservedEntity, type Provenance } from "@src/shared"; /** * Entity Extraction Layer (§5 step 6, §37–§39): merge the same entity seen on several surfaces, * keep provenance per field, compare network vs DOM evidence. */ export interface MergeReport { merged: ObservedEntity[]; network_only: number; dom_only: number; both: number; field_agreements: number; field_conflicts: { fingerprint: string; field: string; network: unknown; dom: unknown }[]; } const COMPARABLE_FIELDS = ["title", "author", "duration", "views_text"] as const; function normalizeForCompare(v: unknown): string { return String(v ?? "").toLowerCase().replace(/\s+/g, " ").trim(); } export function mergeSurfaces(network: ObservedEntity[], dom: ObservedEntity[]): MergeReport { const byFp = new Map(); const report: MergeReport = { merged: [], network_only: 0, dom_only: 0, both: 0, field_agreements: 0, field_conflicts: [] }; for (const e of dom) byFp.set(e.fingerprint, cloneEntity(e)); for (const n of network) { const d = byFp.get(n.fingerprint); if (!d) { byFp.set(n.fingerprint, cloneEntity(n)); report.network_only++; continue; } report.both++; // Combine provenance and fields d.provenance = dedupeProv([...d.provenance, ...n.provenance]); for (const [k, nv] of Object.entries(n.fields)) { const dv = d.fields[k]; if (!dv) { d.fields[k] = nv; continue; } if ((COMPARABLE_FIELDS as readonly string[]).includes(k)) { const a = normalizeForCompare(dv.value); const b = normalizeForCompare(nv.value); if (a && b && (a === b || a.includes(b) || b.includes(a))) report.field_agreements++; else if (a && b) report.field_conflicts.push({ fingerprint: n.fingerprint, field: k, network: nv.value, dom: dv.value }); } d.fields[k] = { value: preferValue(dv, nv), provenance: dedupeProv([...dv.provenance, ...nv.provenance]) }; } d.name = d.name && n.name ? (n.name.length >= d.name.length ? n.name : d.name) : (d.name ?? n.name); d.text = d.text ?? n.text; d.author = d.author ?? n.author; // Structured network numbers beat numbers scraped from card text. d.metrics = { ...(d.metrics ?? {}), ...(n.metrics ?? {}) }; if (Object.keys(d.metrics).length === 0) d.metrics = undefined; d.media = d.media || n.media ? { has_video: !!(d.media?.has_video || n.media?.has_video), has_image: !!(d.media?.has_image || n.media?.has_image), duration_s: d.media?.duration_s ?? n.media?.duration_s, thumbnail_url: d.media?.thumbnail_url ?? n.media?.thumbnail_url } : undefined; d.url = d.url ?? n.url; } report.dom_only = dom.length - report.both; // Re-number refs so the planner sees E1..En in a stable order (DOM order first, then network-only). let i = 0; for (const e of byFp.values()) e.ref = `E${++i}`; report.merged = [...byFp.values()]; return report; } function preferValue(a: Evidenced, b: Evidenced): unknown { const ca = Math.max(...a.provenance.map((p) => p.confidence)); const cb = Math.max(...b.provenance.map((p) => p.confidence)); if (typeof a.value === "string" && typeof b.value === "string" && Math.abs(ca - cb) < 0.1) return a.value.length >= b.value.length ? a.value : b.value; return cb > ca ? b.value : a.value; } function dedupeProv(list: Provenance[]): Provenance[] { const seen = new Map(); for (const p of list) { const k = p.surface + "|" + (p.detail ?? ""); const prev = seen.get(k); if (!prev || prev.confidence < p.confidence) seen.set(k, p); } return [...seen.values()].sort((a, b) => b.confidence - a.confidence); } function cloneEntity(e: ObservedEntity): ObservedEntity { return { ...e, fields: { ...e.fields }, provenance: [...e.provenance], metrics: e.metrics ? { ...e.metrics } : undefined, media: e.media ? { ...e.media } : undefined }; } /** Overall entity confidence = best provenance, boosted when two surfaces agree (§59). */ export function entityConfidence(e: ObservedEntity): number { const surfaces = new Set(e.provenance.map((p) => p.surface)); const best = Math.max(0, ...e.provenance.map((p) => p.confidence)); return clamp01(best + (surfaces.size > 1 ? 0.05 : 0) - (!e.name && !e.text ? 0.2 : 0)); } /** * IdentityResolver (§21) — minimal skeleton. Never merges on names alone: requires at least * one strong signal (same canonical url, cross-link, same platform id). Fuzzy names only add evidence. */ export interface IdentityCandidate { id: string; display_name?: string; urls: string[]; usernames: string[]; } export interface MatchDecision { candidate_a: string; candidate_b: string; match_probability: number; evidence: string[]; merge: boolean; } export function resolveIdentity(a: IdentityCandidate, b: IdentityCandidate): MatchDecision { const evidence: string[] = []; let p = 0; const urlsA = new Set(a.urls.map((u) => u.toLowerCase())); if (b.urls.some((u) => urlsA.has(u.toLowerCase()))) { evidence.push("same canonical url"); p += 0.7; } const uA = new Set(a.usernames.map((u) => u.toLowerCase().replace(/^@/, ""))); if (b.usernames.some((u) => uA.has(u.toLowerCase().replace(/^@/, "")))) { evidence.push("same username"); p += 0.35; } if (a.display_name && b.display_name && a.display_name.toLowerCase() === b.display_name.toLowerCase()) { evidence.push("same display name (weak)"); p += 0.1; } const prob = clamp01(p); return { candidate_a: a.id, candidate_b: b.id, match_probability: prob, evidence, merge: prob >= 0.8 && evidence.some((e) => !e.includes("weak")) }; }