spb/social-runtime-crawler

1import type { Page } from "playwright";2import { createLogger, type Platform } from "@src/shared";3import type { EventBus } from "@src/events";45const log = createLogger("dom");67export interface DomDelta {8  added: number;9  removed: number;10  text_changes: number;11  attr_changes: number;12  regions: Record<string, number>; // landmark/role region → nodes added13  new_videos: number;14  new_articles: number;15  new_links: number;16  modal_opened: boolean;17  url: string;18  at: number;19}2021/**22 * Injected before any page script (§13). Buffers MutationObserver deltas and flushes23 * them to Node every 500 ms through an exposed binding. Never serializes the whole DOM.24 */25const INJECTED = String.raw`26(() => {27  if (window.__srcDomObserverInstalled) return;28  window.__srcDomObserverInstalled = true;29  const state = { added: 0, removed: 0, text_changes: 0, attr_changes: 0, regions: {}, new_videos: 0, new_articles: 0, new_links: 0, modal_opened: false };30  const regionOf = (node) => {31    let el = node.nodeType === 1 ? node : node.parentElement;32    let hops = 0;33    while (el && hops++ < 25) {34      const role = el.getAttribute && (el.getAttribute('role') || '');35      const tag = el.tagName ? el.tagName.toLowerCase() : '';36      if (role === 'main' || tag === 'main') return 'main';37      if (role === 'feed' ) return 'feed';38      if (role === 'dialog' || el.getAttribute && el.getAttribute('aria-modal') === 'true') return 'dialog';39      if (role === 'navigation' || tag === 'nav') return 'navigation';40      if (role === 'complementary' || tag === 'aside') return 'sidebar';41      if (tag === 'header' || role === 'banner') return 'header';42      if (tag === 'ytd-comments' || (el.id && /comment/i.test(el.id))) return 'comments';43      el = el.parentElement;44    }45    return 'unknown';46  };47  const count = (node) => {48    if (node.nodeType !== 1) return;49    const el = node;50    const tag = el.tagName.toLowerCase();51    if (tag === 'video' || el.querySelector && el.querySelector('video')) state.new_videos++;52    if (tag === 'article' || el.getAttribute('role') === 'article' || (el.querySelectorAll && el.querySelectorAll('article,[role=article]').length)) state.new_articles++;53    if (tag === 'a' ) state.new_links++; else if (el.querySelectorAll) state.new_links += Math.min(50, el.querySelectorAll('a[href]').length);54    if (el.getAttribute('role') === 'dialog' || el.getAttribute('aria-modal') === 'true') state.modal_opened = true;55  };56  const obs = new MutationObserver((muts) => {57    for (const m of muts) {58      if (m.type === 'childList') {59        if (m.addedNodes.length) {60          state.added += m.addedNodes.length;61          const r = regionOf(m.target);62          state.regions[r] = (state.regions[r] || 0) + m.addedNodes.length;63          m.addedNodes.forEach(count);64        }65        state.removed += m.removedNodes.length;66      } else if (m.type === 'characterData') state.text_changes++;67      else if (m.type === 'attributes') state.attr_changes++;68    }69  });70  const start = () => obs.observe(document.documentElement, { childList: true, subtree: true, characterData: true, attributes: true, attributeFilter: ['src', 'href', 'aria-expanded', 'aria-hidden', 'hidden', 'class'] });71  if (document.documentElement) start(); else document.addEventListener('DOMContentLoaded', start);72  setInterval(() => {73    if (!state.added && !state.removed && !state.text_changes && !state.attr_changes) return;74    const snapshot = Object.assign({}, state, { url: location.href, at: Date.now() });75    state.added = 0; state.removed = 0; state.text_changes = 0; state.attr_changes = 0; state.regions = {}; state.new_videos = 0; state.new_articles = 0; state.new_links = 0; state.modal_opened = false;76    if (window.__srcEmitDomDelta) window.__srcEmitDomDelta(snapshot);77  }, 500);78})();79`;8081export class DomObserver {82  private deltas: DomDelta[] = [];83  private stepDeltas: DomDelta[] = [];84  private step = 0;85  private installed = false;8687  constructor(private readonly opts: { page: Page; bus: EventBus; sessionId: string; platform: Platform }) {}8889  async attach(): Promise<void> {90    if (this.installed) return;91    const { page } = this.opts;92    await page.exposeBinding("__srcEmitDomDelta", (_src, delta: DomDelta) => this.onDelta(delta));93    await page.addInitScript(INJECTED);94    // For the page already loaded, install immediately too.95    await page.evaluate(INJECTED).catch(() => {});96    this.installed = true;97    log.debug("dom observer attached");98  }99100  beginStep(step: number): void {101    this.step = step;102    this.stepDeltas = [];103  }104105  collectStep(): DomDelta[] {106    return [...this.stepDeltas];107  }108109  private onDelta(delta: DomDelta): void {110    this.deltas.push(delta);111    this.stepDeltas.push(delta);112    if (this.deltas.length > 5000) this.deltas.splice(0, 1000);113    if (delta.added + delta.removed < 3 && delta.text_changes < 3 && !delta.modal_opened) return; // noise114    this.opts.bus.emit({115      event_type: "DOM_CHANGED",116      platform: this.opts.platform,117      session_id: this.opts.sessionId,118      step: this.step,119      payload: {120        added: delta.added,121        removed: delta.removed,122        text_changes: delta.text_changes,123        attr_changes: delta.attr_changes,124        regions: delta.regions,125        new_videos: delta.new_videos,126        new_articles: delta.new_articles,127        new_links: delta.new_links,128        modal_opened: delta.modal_opened,129        url: delta.url,130      },131      provenance: [{ surface: "dom", confidence: 0.9 }],132    });133  }134}135