spb/social-runtime-crawler

1import {2  canonicalUrl,3  parseCount,4  parseDuration,5  shortHash,6  truncate,7  walkJson,8  type EntityType,9  type Evidenced,10  type NetworkFingerprint,11  type ObservedEntity,12  type Platform,13  type Provenance,14  type SchemaProfile,15} from "@src/shared";16import { profileJson } from "./SchemaProfiler.ts";1718export interface CapturedResponse {19  request_id: string;20  url: string;21  method: string;22  status: number;23  content_type: string;24  resource_type: string;25  body_size: number;26  body?: string; // only kept for JSON / GraphQL / small HTML fragments27  post_data?: string;28  timing_ms?: number;29  captured_at: string;30  step?: number;31}3233export interface ClassifiedResponse {34  response: CapturedResponse;35  kind: "json" | "graphql" | "html_fragment" | "media_manifest" | "media_segment" | "image" | "other";36  fingerprint: NetworkFingerprint;37  schema?: SchemaProfile;38  entities: ObservedEntity[]; // generic candidate entities mined from the payload39  confidence: number;40}4142/** Text collapsers let adapters describe how a platform nests rich text (e.g. YouTube `runs`/`simpleText`). */43export type TextCollapser = (v: unknown) => string | undefined;4445export interface ClassifierHints {46  platform: Platform;47  textCollapsers?: TextCollapser[];48  /** Key names (case-insensitive substrings) that strongly denote an entity id on this platform. */49  idKeys?: Record<string, EntityType>;50  /** GraphQL `__typename` vocabulary → entity type (generic GraphQL convention, values are platform knowledge). */51  typenames?: Record<string, EntityType>;52  /** Build a canonical page URL from a platform id. */53  urlForId?: (type: EntityType, id: string) => string | undefined;54}5556export function pathPattern(url: string): { hostname: string; path_pattern: string } {57  try {58    const u = new URL(url);59    const path = u.pathname60      .split("/")61      .map((seg) => (/^\d+$/.test(seg) || /^[A-Za-z0-9_-]{11,}$/.test(seg) || /^t[0-9]_[a-z0-9]+$/.test(seg) ? "*" : seg))62      .join("/");63    return { hostname: u.hostname, path_pattern: path || "/" };64  } catch {65    return { hostname: "", path_pattern: url };66  }67}6869export function classifyKind(r: CapturedResponse): ClassifiedResponse["kind"] {70  const ct = r.content_type.toLowerCase();71  const u = r.url.toLowerCase();72  if (/\.m3u8|\.mpd|mpegurl|dash\+xml|manifest/.test(u + " " + ct)) return "media_manifest";73  if (/videoplayback|\.ts(\?|$)|\.m4s|\.mp4|video\/|audio\//.test(u + " " + ct)) return "media_segment";74  if (ct.startsWith("image/")) return "image";75  // Anti-hijack prefixes (`for (;;);`, `)]}'`) hide JSON behind a javascript content-type.76  const bodyStart = r.body?.replace(/^\)\]\}'\s*/, "").replace(/^for \(;;\);/, "").trimStart().slice(0, 1);77  if (ct.includes("json") || bodyStart === "{" || bodyStart === "[" || /\/graphql\/?(\?|$)/i.test(u)) {78    if (/graphql/i.test(u) || (r.post_data && /"query"\s*:|operationName|doc_id|fb_api_req_friendly_name/.test(r.post_data))) return "graphql";79    return "json";80  }81  if (ct.includes("text/html") && r.resource_type !== "document") return "html_fragment";82  return "other";83}8485function graphqlOperation(r: CapturedResponse): string | undefined {86  if (!r.post_data) return undefined;87  const m = r.post_data.match(/(?:"operationName"\s*:\s*"|fb_api_req_friendly_name=)([A-Za-z0-9_]+)/);88  return m?.[1];89}9091function looseJson(body: string): unknown | undefined {92  // Some platforms prefix JSON with anti-hijack tokens (")]}'", "for (;;);") or send NDJSON.93  const cleaned = body.replace(/^\)\]\}'\s*/, "").replace(/^for \(;;\);/, "").trim();94  try {95    return JSON.parse(cleaned);96  } catch {97    const lines = cleaned.split("\n").filter((l) => l.trim().startsWith("{"));98    if (lines.length > 1) {99      const parsed = lines.map((l) => {100        try {101          return JSON.parse(l);102        } catch {103          return undefined;104        }105      }).filter(Boolean);106      return parsed.length ? parsed : undefined;107    }108    return undefined;109  }110}111112/**113 * Generic JSON entity miner: find objects that carry an id-like field plus some114 * displayable content, without knowing the platform's property names.115 */116export function mineEntities(root: unknown, hints: ClassifierHints, step?: number): ObservedEntity[] {117  const collapsers = hints.textCollapsers ?? [];118  const collapse = (v: unknown): string | undefined => {119    if (typeof v === "string") return v;120    if (typeof v === "number") return String(v);121    for (const c of collapsers) {122      const s = c(v);123      if (s) return s;124    }125    return undefined;126  };127  const idKeyEntries = Object.entries(hints.idKeys ?? {}).map(([k, t]) => [k.toLowerCase(), t] as const);128  const found = new Map<string, ObservedEntity>();129  let refCounter = 0;130131  walkJson(root, (v, path) => {132    if (!v || typeof v !== "object" || Array.isArray(v)) return;133    const obj = v as Record<string, unknown>;134    const keys = Object.keys(obj);135    // 0) GraphQL __typename is the strongest type evidence when the adapter knows the vocabulary136    const typename = typeof obj.__typename === "string" ? hints.typenames?.[obj.__typename] : undefined;137    // 1) find an id138    let idKey: string | undefined;139    let idType: EntityType | undefined = typename;140    if (typename && typeof obj.id === "string" && obj.id.length >= 4) idKey = "id";141    for (const k of keys) {142      if (idKey) break;143      const kl = k.toLowerCase();144      const hint = idKeyEntries.find(([hk]) => kl === hk);145      if (hint && (typeof obj[k] === "string" || typeof obj[k] === "number")) {146        idKey = k;147        idType = hint[1];148        break;149      }150    }151    if (!idKey) {152      for (const k of keys) {153        if (/(^id$|Id$|_id$|^videoId$|^channelId$|^postId$|^name$)/.test(k) && k !== "name" && typeof obj[k] === "string" && /^[A-Za-z0-9_-]{4,64}$/.test(obj[k] as string)) {154          idKey = k;155          break;156        }157      }158    }159    if (!idKey) return;160    const id = String(obj[idKey]);161162    // 2) displayable content nearby (shallow)163    const pick = (re: RegExp) => {164      for (const k of keys) if (re.test(k)) {165        const s = collapse(obj[k]);166        if (s && s.length > 0) return s;167      }168      return undefined;169    };170    const title = pick(/^(title|headline|name|displayName|fullName|display_name)$/i);171    const text = pick(/^(text|body|caption|description|selftext|descriptionSnippet|content|message|savable_description|body_text)$/i);172    const author = pick(/^(author|ownerText|shortBylineText|longBylineText|author_name|username|channelName|user_name|screen_name|handle|owner|actors?)$/i) ?? nestedName(obj, /^(owner|author|actors?|creator|profile|user|page|video_owner|feedback_owner)$/i, collapse);173    const thumb = (() => {174      let out: string | undefined;175      walkJson(obj, (x, p) => {176        if (out) return false;177        if (p.length > 4) return false;178        if (typeof x === "string" && /^https?:\/\/.*(\.(jpe?g|png|webp)|ytimg|thumbnail|preview|avatar)/i.test(x) && /(thumb|image|img|picture|avatar|preview|url)/i.test(p.join("."))) out = x;179      });180      return out;181    })();182    const durationText = pick(/^(lengthText|duration|length|durationText|video_duration)$/i);183    const viewsText = pick(/^(viewCountText|shortViewCountText|views|view_count|viewCount|ups|score|num_comments|likeCount|like_count)$/i);184185    if (!title && !text && !idType) return; // an id alone is not an entity186187    // 3) type guess188    let type: EntityType = idType ?? "post";189    if (!idType) {190      if (/video/i.test(idKey) || durationText || /video|shorts|watch/i.test(keys.join(" "))) type = "video";191      else if (/channel|user|author|owner|profile/i.test(idKey) || (title && !text && /subscriber|follower|handle|username/i.test(keys.join(" ")))) type = "profile";192      else if (/comment|reply/i.test(idKey) || /parent_id|replyCount|depth/i.test(keys.join(" "))) type = "comment";193    }194195    const url = pick(/^(url|permalink|canonicalUrl|href|link|permalink_url|wwwURL|profile_url|share_url)$/i) ?? hints.urlForId?.(type, id) ?? undefined;196    const fingerprint = `${hints.platform}:${type}:${id}`;197    if (found.has(fingerprint)) return;198199    const prov: Provenance[] = [{ surface: "network", confidence: idType ? 0.95 : 0.7, detail: path.join(".") }];200    const ev = (value: unknown): Evidenced => ({ value, provenance: prov });201    const fields: Record<string, Evidenced> = { platform_id: ev(id) };202    if (title) fields.title = ev(title);203    if (text) fields.text = ev(text);204    if (author) fields.author = ev(author);205    if (thumb) fields.thumbnail_url = ev(thumb);206    if (durationText) fields.duration = ev(durationText);207    if (viewsText) fields.views_text = ev(viewsText);208209    const metrics: ObservedEntity["metrics"] = {};210    const views = parseCount(viewsText);211    if (views !== undefined) {212      if (type === "video") metrics.views = views;213      else metrics.score = views;214    }215216    found.set(fingerprint, {217      ref: `N${++refCounter}`,218      type,219      platform: hints.platform,220      platform_id: id,221      url: url ? canonicalUrl(url.startsWith("http") ? url : `https://${hints.platform === "x" ? "x.com" : hints.platform + ".com"}${url}`) : undefined,222      name: title ? truncate(title, 200) : undefined,223      text: text ? truncate(text, 500) : undefined,224      author: author ? truncate(author, 120) : undefined,225      metrics: Object.keys(metrics).length ? metrics : undefined,226      media: type === "video" ? { has_video: true, has_image: !!thumb, duration_s: parseDuration(durationText), thumbnail_url: thumb } : thumb ? { has_video: false, has_image: true, thumbnail_url: thumb } : undefined,227      context: `network ${path.slice(0, 4).join(".")}`,228      fields,229      provenance: prov,230      fingerprint,231    });232    if (found.size >= 400) return false;233  });234235  return [...found.values()];236}237238/** Author objects are often nested: `{ owner: { name, __typename: "Page" } }` → "name". */239function nestedName(obj: Record<string, unknown>, keyRe: RegExp, collapse: (v: unknown) => string | undefined): string | undefined {240  for (const [k, v] of Object.entries(obj)) {241    if (!keyRe.test(k)) continue;242    const candidates = Array.isArray(v) ? v : [v];243    for (const c of candidates) {244      if (!c || typeof c !== "object") continue;245      const o = c as Record<string, unknown>;246      const s = collapse(o.name) ?? collapse(o.title) ?? collapse(o.display_name) ?? collapse(o.username);247      if (s) return s;248    }249  }250  return undefined;251}252253export function classifyResponse(r: CapturedResponse, hints: ClassifierHints): ClassifiedResponse {254  const kind = classifyKind(r);255  const { hostname, path_pattern } = pathPattern(r.url);256  const base: NetworkFingerprint = {257    hostname,258    path_pattern,259    method: r.method,260    content_type: r.content_type.split(";")[0] ?? "",261    response_shape_hash: "",262    is_graphql: kind === "graphql",263    graphql_operation: kind === "graphql" ? graphqlOperation(r) : undefined,264    observed_entity_types: [],265  };266  let schema: SchemaProfile | undefined;267  let entities: ObservedEntity[] = [];268  let confidence = 0.3;269270  if ((kind === "json" || kind === "graphql") && r.body) {271    const parsed = looseJson(r.body);272    if (parsed !== undefined) {273      schema = profileJson(parsed);274      base.response_shape_hash = schema.shape_hash;275      entities = mineEntities(parsed, hints, r.step);276      const types = new Set<EntityType>();277      for (const e of entities) types.add(e.type);278      for (const c of schema.candidate_entity_types) if (c.confidence >= 0.6) types.add(c.type);279      base.observed_entity_types = [...types];280      confidence = entities.length ? 0.9 : schema.candidate_entity_types.length ? 0.6 : 0.4;281    }282  } else {283    base.response_shape_hash = shortHash(`${kind}:${hostname}:${path_pattern}:${base.content_type}`, 16);284    if (kind === "media_manifest" || kind === "media_segment") confidence = 0.8;285  }286  return { response: r, kind, fingerprint: base, schema, entities, confidence };287}288