spb/social-runtime-crawler

1import type { PageClassification, PageType } from "@src/shared";2import type { DomSnapshot } from "./dom/PageSummarizer.ts";34/** A URL-pattern hint supplied by a platform adapter (§58). */5export interface PageTypeHint {6  pattern: RegExp; // tested against pathname + search7  page_type: PageType;8  confidence: number;9}1011/**12 * Generic page classifier combining URL hints, DOM landmarks and content density (§58–§60).13 * Returns UNKNOWN with low confidence rather than throwing — unknown pages are expected.14 */15export function classifyPage(snapshot: DomSnapshot, hints: PageTypeHint[] = []): PageClassification {16  const signals: string[] = [];17  let url: URL | undefined;18  try {19    url = new URL(snapshot.url);20  } catch {21    /* ignore */22  }23  const scores = new Map<PageType, number>();24  const bump = (t: PageType, c: number, why: string) => {25    scores.set(t, Math.max(scores.get(t) ?? 0, c));26    signals.push(`${why}→${t}`);27  };2829  if (snapshot.has_login_form) bump("LOGIN", 0.9, "password field");3031  if (url) {32    const target = url.pathname + url.search;33    for (const h of hints) if (h.pattern.test(target)) bump(h.page_type, h.confidence, `url:${h.pattern.source.slice(0, 30)}`);34    if (target === "/" || target === "") bump("HOME_FEED", 0.6, "root path");35    if (/search|results|\?q=|\?search_query=/i.test(target)) bump("SEARCH_RESULTS", 0.7, "search in url");36  }3738  const links = snapshot.candidates.filter((c) => c.kind === "link");39  const articles = snapshot.candidates.filter((c) => c.kind === "article");40  const mainLinks = links.filter((c) => c.region === "main" || c.region === "feed").length;41  const hasVideo = snapshot.video_elements.some((v) => v.width > 0 || v.duration);42  const commentsRegion = snapshot.candidates.some((c) => c.region === "comments");4344  if (hasVideo && snapshot.video_elements.length === 1 && snapshot.landmarks.length > 2) bump("VIDEO_DETAIL", 0.6, "single video element");45  if (commentsRegion && (hasVideo || articles.length <= 2)) bump(hasVideo ? "VIDEO_DETAIL" : "POST_DETAIL", 0.55, "comments region");46  if (articles.length >= 5 && !hasVideo) bump("HOME_FEED", 0.5, `${articles.length} articles`);47  if (mainLinks >= 15 && articles.length >= 5 && scores.get("SEARCH_RESULTS") === undefined) bump("HOME_FEED", 0.45, "dense feed");48  if (snapshot.candidates.some((c) => c.kind === "search" && c.visible) && snapshot.candidates.some((c) => c.kind === "heading" && /result|résultat/i.test(c.text))) bump("SEARCH_RESULTS", 0.65, "results heading");4950  if (scores.size === 0) return { page_type: "UNKNOWN", confidence: 0.2, signals: ["no signal"] };51  const [best, conf] = [...scores.entries()].sort((a, b) => b[1] - a[1])[0]!;52  return { page_type: best, confidence: Math.min(0.99, conf), signals: signals.slice(0, 12) };53}54