spb/social-runtime-crawler

1import { jaccard, tokenize, type EntityType, type ObservedEntity, type RelationType } from "@src/shared";23/**4 * Social World Model (§23): an in-memory graph of everything observed in this session.5 * Guides navigation (visited set, novelty) and becomes the seed of the persisted graph.6 */7export interface WorldNode {8  fingerprint: string;9  type: EntityType;10  name?: string;11  url?: string;12  platform_id?: string;13  first_seen_step: number;14  last_seen_step: number;15  seen_count: number;16  visited: boolean; // we navigated to it17  surfaces: Set<string>;18  tokens: Set<string>;19}2021export interface WorldEdge {22  from: string;23  to: string;24  type: RelationType;25  step: number;26}2728export class WorldModel {29  readonly nodes = new Map<string, WorldNode>();30  readonly edges: WorldEdge[] = [];31  readonly visitedUrls = new Set<string>();32  private edgeKeys = new Set<string>();3334  /** Register observed entities; returns those never seen before. */35  observe(entities: ObservedEntity[], step: number, discoveredFrom?: string): ObservedEntity[] {36    const fresh: ObservedEntity[] = [];37    for (const e of entities) {38      const n = this.nodes.get(e.fingerprint);39      if (n) {40        n.last_seen_step = step;41        n.seen_count++;42        if (!n.name && e.name) n.name = e.name;43        if (!n.url && e.url) n.url = e.url;44        for (const p of e.provenance) n.surfaces.add(p.surface);45        if (e.name || e.text) n.tokens = tokenize(`${e.name ?? ""} ${e.text ?? ""} ${e.author ?? ""}`);46      } else {47        this.nodes.set(e.fingerprint, {48          fingerprint: e.fingerprint,49          type: e.type,50          name: e.name,51          url: e.url,52          platform_id: e.platform_id,53          first_seen_step: step,54          last_seen_step: step,55          seen_count: 1,56          visited: false,57          surfaces: new Set(e.provenance.map((p) => p.surface)),58          tokens: tokenize(`${e.name ?? ""} ${e.text ?? ""} ${e.author ?? ""}`),59        });60        fresh.push(e);61        if (discoveredFrom) this.relate(discoveredFrom, e.fingerprint, "DISCOVERED_FROM", step);62      }63      if (e.author && e.author_url) {64        // author edge when we know the author's page65        this.relate(`author:${e.author_url}`, e.fingerprint, "AUTHORED", step);66      }67    }68    return fresh;69  }7071  relate(from: string, to: string, type: RelationType, step: number): void {72    const k = `${from}→${to}:${type}`;73    if (this.edgeKeys.has(k)) return;74    this.edgeKeys.add(k);75    this.edges.push({ from, to, type, step });76  }7778  markVisited(fingerprint: string | undefined, url: string | undefined): void {79    if (fingerprint) {80      const n = this.nodes.get(fingerprint);81      if (n) n.visited = true;82    }83    if (url) this.visitedUrls.add(url);84  }8586  isVisited(e: ObservedEntity): boolean {87    return (this.nodes.get(e.fingerprint)?.visited ?? false) || (!!e.url && this.visitedUrls.has(e.url));88  }8990  /** Lexical novelty: 1 − max Jaccard similarity to what we already know (embedding-ready seam, §27). */91  novelty(e: ObservedEntity): number {92    const t = tokenize(`${e.name ?? ""} ${e.text ?? ""} ${e.author ?? ""}`);93    if (t.size === 0) return 0.5;94    let max = 0;95    let checked = 0;96    for (const n of this.nodes.values()) {97      if (n.fingerprint === e.fingerprint) continue;98      if (n.tokens.size === 0) continue;99      const s = jaccard(t, n.tokens);100      if (s > max) max = s;101      if (++checked > 2000) break;102    }103    return 1 - max;104  }105106  stats() {107    const byType: Record<string, number> = {};108    let visited = 0;109    let multi = 0;110    for (const n of this.nodes.values()) {111      byType[n.type] = (byType[n.type] ?? 0) + 1;112      if (n.visited) visited++;113      if (n.surfaces.size > 1) multi++;114    }115    return { nodes: this.nodes.size, edges: this.edges.length, visited, multi_surface: multi, by_type: byType };116  }117118  toJSON() {119    return {120      nodes: [...this.nodes.values()].map((n) => ({ ...n, surfaces: [...n.surfaces], tokens: undefined })),121      edges: this.edges,122    };123  }124}125