spb/social-runtime-crawler

1import type { Page } from "playwright";2import {3  canonicalUrl,4  parseCount,5  parseDuration,6  shortHash,7  truncate,8  type EntityType,9  type Evidenced,10  type ObservedEntity,11  type PageType,12  type Platform,13  type Provenance,14  type SemanticAction,15} from "@src/shared";16import type { ClassifierHints, DomCandidate, DomSnapshot, PageTypeHint } from "@src/observers";1718/**19 * SocialPlatformAdapter (§56/§57). Adapters hold the *platform-specific* knowledge:20 * page recognition, semantic labels, entity URL grammar, normalization quirks.21 * Everything generic (browser lifecycle, observers, planner, storage) lives outside.22 */23export interface UrlEntityRule {24  pattern: RegExp; // tested against pathname+search of an absolute URL25  type: EntityType;26  /** Extract platform id from the match. */27  id: (m: RegExpMatchArray, url: URL) => string | undefined;28  confidence: number;29}3031export interface SocialPlatformAdapter {32  readonly platform: Platform;33  readonly homeUrl: string;34  readonly hosts: RegExp;35  readonly pageTypeHints: PageTypeHint[];36  readonly urlEntityRules: UrlEntityRule[];37  readonly classifierHints: ClassifierHints;38  /** Build a search URL for native platform search (§43). */39  searchUrl(query: string): string;40  /** Extra filters when turning DOM links into entities (e.g. skip menu links). */41  isNoiseLink?(href: URL, text: string): boolean;42  /** Human labels for the planner. */43  entityLabel(type: EntityType): string;44  /** Optional per-page tweaks after generic extraction. */45  refineEntities?(entities: ObservedEntity[], snapshot: DomSnapshot): ObservedEntity[];46  /** Expected minimum entities on a known page type — used for degradation detection (§32). */47  expectedEntities(pageType: PageType): number;48  /** Optional: dismiss consent / cookie dialogs (read-only UI hygiene). */49  dismissOverlays?(page: Page): Promise<void>;50}5152const OPEN_ACTION: Partial<Record<EntityType, SemanticAction["type"]>> = {53  video: "OPEN_VIDEO",54  post: "OPEN_POST",55  comment: "OPEN_COMMENTS",56  profile: "OPEN_PROFILE",57  channel: "OPEN_CHANNEL",58  page: "OPEN_PAGE",59  community: "OPEN_PAGE",60  person: "OPEN_PROFILE",61  organization: "OPEN_PAGE",62};6364/**65 * Generic DOM → entity extraction, driven by the adapter's URL grammar (§57: "feed identification,66 * known entity classes" are platform knowledge; the walk itself is generic).67 */68export function entitiesFromDom(snapshot: DomSnapshot, adapter: SocialPlatformAdapter): ObservedEntity[] {69  const out = new Map<string, ObservedEntity>();70  let n = 0;71  for (const c of snapshot.candidates) {72    if (!c.href) continue;73    let url: URL;74    try {75      url = new URL(c.href);76    } catch {77      continue;78    }79    if (!adapter.hosts.test(url.hostname)) continue;80    if (adapter.isNoiseLink?.(url, c.text)) continue;81    const target = url.pathname + url.search;82    for (const rule of adapter.urlEntityRules) {83      const m = target.match(rule.pattern);84      if (!m) continue;85      const id = rule.id(m, url);86      if (!id) break;87      const fingerprint = `${adapter.platform}:${rule.type}:${id}`;88      const prov: Provenance[] = [{ surface: "dom", confidence: rule.confidence, detail: `${c.kind} in ${c.region}` }];89      const ev = (value: unknown): Evidenced => ({ value, provenance: prov });90      const existing = out.get(fingerprint);91      const text = c.text && c.text.length > 1 ? dedupeRepeatedText(c.text) : undefined;92      if (existing) {93        // Merge: a longer text is a better title for the same link target.94        if (text && (!existing.name || text.length > existing.name.length) && !/^\d+:\d\d/.test(text)) {95          existing.name = truncate(text, 200);96          existing.fields.title = ev(text);97        }98        if (c.meta?.duration && !existing.media?.duration_s) existing.media = { ...(existing.media ?? { has_video: rule.type === "video", has_image: false }), duration_s: parseDuration(c.meta.duration) };99        if (c.meta?.views && !existing.metrics?.views) existing.metrics = { ...(existing.metrics ?? {}), views: parseCount(c.meta.views) };100        continue;101      }102      const fields: Record<string, Evidenced> = { platform_id: ev(id), url: ev(url.toString()) };103      if (text) fields.title = ev(text);104      if (c.meta?.author) fields.author = ev(c.meta.author);105      if (c.meta?.duration) fields.duration = ev(c.meta.duration);106      if (c.meta?.views) fields.views_text = ev(c.meta.views);107      if (c.meta?.published) fields.published_text = ev(c.meta.published);108      const metrics: ObservedEntity["metrics"] = {};109      const views = parseCount(c.meta?.views);110      if (views !== undefined) (rule.type === "video" ? (metrics.views = views) : (metrics.score = views));111      const comments = parseCount(c.meta?.comments);112      if (comments !== undefined) metrics.comments = comments;113      out.set(fingerprint, {114        ref: `D${++n}`,115        type: rule.type,116        platform: adapter.platform,117        platform_id: id,118        url: canonicalUrl(url.toString()),119        name: text ? truncate(text, 200) : undefined,120        author: c.meta?.author ? truncate(c.meta.author, 120) : undefined,121        metrics: Object.keys(metrics).length ? metrics : undefined,122        media: rule.type === "video" ? { has_video: true, has_image: false, duration_s: parseDuration(c.meta?.duration) } : undefined,123        published_text: c.meta?.published,124        context: `${c.region}${c.visible ? " (visible)" : ""} #${c.index}`,125        fields,126        provenance: prov,127        fingerprint,128      });129      break;130    }131  }132  const list = [...out.values()];133  return adapter.refineEntities ? adapter.refineEntities(list, snapshot) : list;134}135136/** Turn entities + DOM controls into the semantic action list the planner may choose from (§25). */137export function buildActions(entities: ObservedEntity[], snapshot: DomSnapshot, adapter: SocialPlatformAdapter, opts: { query?: string; maxOpen?: number; visited: Set<string> } ): SemanticAction[] {138  const actions: SemanticAction[] = [];139  let n = 0;140  const id = () => `A${++n}`;141  const openable = entities142    .filter((e) => e.url && OPEN_ACTION[e.type] && !opts.visited.has(e.fingerprint))143    .slice(0, opts.maxOpen ?? 25);144  for (const e of openable) {145    actions.push({146      id: id(),147      type: OPEN_ACTION[e.type]!,148      target_ref: e.ref,149      target_url: e.url,150      label: `Open ${adapter.entityLabel(e.type)} ${e.ref}: ${truncate(e.name ?? e.platform_id ?? e.url, 70)}`,151      cost: 2,152    });153  }154  const canScroll = snapshot.scroll.y + snapshot.scroll.viewport < snapshot.scroll.height - 50;155  if (canScroll) actions.push({ id: id(), type: "SCROLL_DOWN", label: "Scroll down to reveal more content", cost: 1 });156  if (snapshot.scroll.y > 0) actions.push({ id: id(), type: "SCROLL_UP", label: "Scroll up", cost: 1 });157  const expandButtons = snapshot.candidates.filter((c) => c.kind === "button" && c.visible).slice(0, 3);158  for (const b of expandButtons) actions.push({ id: id(), type: "EXPAND", label: `Expand: ${truncate(b.text, 60)}`, target_url: b.locator_hint, cost: 1.5 });159  if (snapshot.video_elements.length && !snapshot.video_elements.some((v) => v.playing)) actions.push({ id: id(), type: "PLAY_VIDEO", label: "Play the visible video (to observe media delivery)", cost: 1.5 });160  if (opts.query) actions.push({ id: id(), type: "SEARCH", query: opts.query, label: `Search the platform for “${opts.query}”`, cost: 2 });161  actions.push({ id: id(), type: "BACK", label: "Go back", cost: 1.5 });162  actions.push({ id: id(), type: "RETURN_TO_FEED", target_url: adapter.homeUrl, label: "Return to the home feed", cost: 2 });163  actions.push({ id: id(), type: "END_SESSION", label: "End the session (nothing valuable left)", cost: 0.5 });164  return actions;165}166167/** Link text often repeats its accessible label ("Mila Mila", "Title Title 12:34") — keep one copy. */168export function dedupeRepeatedText(text: string): string {169  const t = text.replace(/\s+/g, " ").trim();170  const half = Math.floor(t.length / 2);171  for (let cut = half; cut >= 4; cut--) {172    const a = t.slice(0, cut).trim();173    const rest = t.slice(cut).trim();174    if (rest.startsWith(a) && (rest.length === a.length || rest.length - a.length < 40)) return rest.length === a.length ? a : `${a} ${rest.slice(a.length).trim()}`.trim();175  }176  return t;177}178179export function pageFingerprint(url: string, entities: ObservedEntity[]): string {180  const ids = entities.map((e) => e.fingerprint).sort().slice(0, 40).join("|");181  return shortHash(canonicalUrl(url) + "::" + ids, 16);182}183