spb/social-runtime-crawler

1import { clamp01, type Evidenced, type ObservedEntity, type Provenance } from "@src/shared";23/**4 * Entity Extraction Layer (§5 step 6, §37–§39): merge the same entity seen on several surfaces,5 * keep provenance per field, compare network vs DOM evidence.6 */78export interface MergeReport {9  merged: ObservedEntity[];10  network_only: number;11  dom_only: number;12  both: number;13  field_agreements: number;14  field_conflicts: { fingerprint: string; field: string; network: unknown; dom: unknown }[];15}1617const COMPARABLE_FIELDS = ["title", "author", "duration", "views_text"] as const;1819function normalizeForCompare(v: unknown): string {20  return String(v ?? "").toLowerCase().replace(/\s+/g, " ").trim();21}2223export function mergeSurfaces(network: ObservedEntity[], dom: ObservedEntity[]): MergeReport {24  const byFp = new Map<string, ObservedEntity>();25  const report: MergeReport = { merged: [], network_only: 0, dom_only: 0, both: 0, field_agreements: 0, field_conflicts: [] };2627  for (const e of dom) byFp.set(e.fingerprint, cloneEntity(e));28  for (const n of network) {29    const d = byFp.get(n.fingerprint);30    if (!d) {31      byFp.set(n.fingerprint, cloneEntity(n));32      report.network_only++;33      continue;34    }35    report.both++;36    // Combine provenance and fields37    d.provenance = dedupeProv([...d.provenance, ...n.provenance]);38    for (const [k, nv] of Object.entries(n.fields)) {39      const dv = d.fields[k];40      if (!dv) {41        d.fields[k] = nv;42        continue;43      }44      if ((COMPARABLE_FIELDS as readonly string[]).includes(k)) {45        const a = normalizeForCompare(dv.value);46        const b = normalizeForCompare(nv.value);47        if (a && b && (a === b || a.includes(b) || b.includes(a))) report.field_agreements++;48        else if (a && b) report.field_conflicts.push({ fingerprint: n.fingerprint, field: k, network: nv.value, dom: dv.value });49      }50      d.fields[k] = { value: preferValue(dv, nv), provenance: dedupeProv([...dv.provenance, ...nv.provenance]) };51    }52    d.name = d.name && n.name ? (n.name.length >= d.name.length ? n.name : d.name) : (d.name ?? n.name);53    d.text = d.text ?? n.text;54    d.author = d.author ?? n.author;55    // Structured network numbers beat numbers scraped from card text.56    d.metrics = { ...(d.metrics ?? {}), ...(n.metrics ?? {}) };57    if (Object.keys(d.metrics).length === 0) d.metrics = undefined;58    d.media = d.media || n.media ? { has_video: !!(d.media?.has_video || n.media?.has_video), has_image: !!(d.media?.has_image || n.media?.has_image), duration_s: d.media?.duration_s ?? n.media?.duration_s, thumbnail_url: d.media?.thumbnail_url ?? n.media?.thumbnail_url } : undefined;59    d.url = d.url ?? n.url;60  }61  report.dom_only = dom.length - report.both;62  // Re-number refs so the planner sees E1..En in a stable order (DOM order first, then network-only).63  let i = 0;64  for (const e of byFp.values()) e.ref = `E${++i}`;65  report.merged = [...byFp.values()];66  return report;67}6869function preferValue(a: Evidenced, b: Evidenced): unknown {70  const ca = Math.max(...a.provenance.map((p) => p.confidence));71  const cb = Math.max(...b.provenance.map((p) => p.confidence));72  if (typeof a.value === "string" && typeof b.value === "string" && Math.abs(ca - cb) < 0.1) return a.value.length >= b.value.length ? a.value : b.value;73  return cb > ca ? b.value : a.value;74}7576function dedupeProv(list: Provenance[]): Provenance[] {77  const seen = new Map<string, Provenance>();78  for (const p of list) {79    const k = p.surface + "|" + (p.detail ?? "");80    const prev = seen.get(k);81    if (!prev || prev.confidence < p.confidence) seen.set(k, p);82  }83  return [...seen.values()].sort((a, b) => b.confidence - a.confidence);84}8586function cloneEntity(e: ObservedEntity): ObservedEntity {87  return { ...e, fields: { ...e.fields }, provenance: [...e.provenance], metrics: e.metrics ? { ...e.metrics } : undefined, media: e.media ? { ...e.media } : undefined };88}8990/** Overall entity confidence = best provenance, boosted when two surfaces agree (§59). */91export function entityConfidence(e: ObservedEntity): number {92  const surfaces = new Set(e.provenance.map((p) => p.surface));93  const best = Math.max(0, ...e.provenance.map((p) => p.confidence));94  return clamp01(best + (surfaces.size > 1 ? 0.05 : 0) - (!e.name && !e.text ? 0.2 : 0));95}9697/**98 * IdentityResolver (§21) — minimal skeleton. Never merges on names alone: requires at least99 * one strong signal (same canonical url, cross-link, same platform id). Fuzzy names only add evidence.100 */101export interface IdentityCandidate {102  id: string;103  display_name?: string;104  urls: string[];105  usernames: string[];106}107108export interface MatchDecision {109  candidate_a: string;110  candidate_b: string;111  match_probability: number;112  evidence: string[];113  merge: boolean;114}115116export function resolveIdentity(a: IdentityCandidate, b: IdentityCandidate): MatchDecision {117  const evidence: string[] = [];118  let p = 0;119  const urlsA = new Set(a.urls.map((u) => u.toLowerCase()));120  if (b.urls.some((u) => urlsA.has(u.toLowerCase()))) {121    evidence.push("same canonical url");122    p += 0.7;123  }124  const uA = new Set(a.usernames.map((u) => u.toLowerCase().replace(/^@/, "")));125  if (b.usernames.some((u) => uA.has(u.toLowerCase().replace(/^@/, "")))) {126    evidence.push("same username");127    p += 0.35;128  }129  if (a.display_name && b.display_name && a.display_name.toLowerCase() === b.display_name.toLowerCase()) {130    evidence.push("same display name (weak)");131    p += 0.1;132  }133  const prob = clamp01(p);134  return { candidate_a: a.id, candidate_b: b.id, match_probability: prob, evidence, merge: prob >= 0.8 && evidence.some((e) => !e.includes("weak")) };135}136