spb/social-runtime-crawler

1import {2  shortHash,3  type EntityType,4  type SchemaField,5  type SchemaProfile,6  type SemanticFieldGuess,7  type SemanticFieldKind,8} from "@src/shared";910/**11 * SchemaProfiler (§11, §12): infers structure and field semantics of a JSON payload12 * without assuming property names. Purely deterministic (Tier 1).13 */1415const ID_KEY = /(^|_|\b)(id|ids|uid|guid|key|pk)$|Id$|ID$|_id$/;16const USERNAME_KEY = /(user|screen)?name$|handle|login|slug|^author$|owner/i;17const TITLE_KEY = /title|headline|subject/i;18const TEXT_KEY = /text|body|caption|description|content|message|selftext|snippet/i;19const URL_KEY = /url|href|link|permalink|uri/i;20const MEDIA_KEY = /video|stream|manifest|playback|media|mp4|hls|dash|audio/i;21const THUMB_KEY = /thumb|thumbnail|poster|preview|avatar|icon|image|img|picture|photo/i;22const TIME_KEY = /time|date|created|published|updated|_at$|timestamp|utc/i;23const COUNT_KEY = /count|views|likes|score|ups|downs|comments|shares|subscribers|followers|favorites|reposts|replies|num_/i;24const DURATION_KEY = /duration|length/i;25const CURSOR_KEY = /cursor|continuation|token|after|before|next|page_?info|offset/i;2627function typeOf(v: unknown): string {28  if (v === null) return "null";29  if (Array.isArray(v)) return "array";30  return typeof v;31}3233export function guessFieldSemantics(key: string, values: unknown[]): SemanticFieldGuess[] {34  const guesses: Map<SemanticFieldKind, number> = new Map();35  const add = (k: SemanticFieldKind, c: number) => guesses.set(k, Math.max(guesses.get(k) ?? 0, c));36  const strings = values.filter((v): v is string => typeof v === "string");37  const numbers = values.filter((v): v is number => typeof v === "number");38  const bools = values.filter((v) => typeof v === "boolean");39  const n = values.length || 1;4041  if (bools.length / n > 0.8) add("boolean", 0.95);4243  // value-shape signals44  const urlRatio = strings.filter((s) => /^https?:\/\//.test(s)).length / n;45  if (urlRatio > 0.7) {46    add("url", 0.85);47    const mediaRatio = strings.filter((s) => /\.(m3u8|mpd|mp4|webm|m4a|mp3)(\?|$)|videoplayback|\/video\//i.test(s)).length / n;48    // Value evidence is more specific than the key name: a URL whose values look like images *is* a thumbnail url.49    if (mediaRatio > 0.5) add("media_url", 0.93);50    const thumbRatio = strings.filter((s) => /\.(jpe?g|png|webp|gif)(\?|$)|thumbnail|ytimg|preview|avatar/i.test(s)).length / n;51    if (thumbRatio > 0.5) add("thumbnail_url", 0.92);52  }53  const handleRatio = strings.filter((s) => /^@[\w.]{2,40}$/.test(s) || /^u\/[\w-]+$/.test(s)).length / n;54  if (handleRatio > 0.6) add("username", 0.9);55  const isoRatio = strings.filter((s) => /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(s)).length / n;56  if (isoRatio > 0.6) add("timestamp", 0.95);57  const epochRatio = numbers.filter((x) => x > 1_000_000_000 && x < 4_000_000_000).length / n;58  if (epochRatio > 0.6) add("timestamp", 0.75);59  const durationRatio = strings.filter((s) => /^(\d+:)?\d{1,2}:\d{2}$/.test(s) || /^PT\d/.test(s)).length / n;60  if (durationRatio > 0.6) add("duration", 0.9);61  const idLike = strings.filter((s) => /^[A-Za-z0-9_-]{6,64}$/.test(s) && !/\s/.test(s) && !/^[a-z]+$/.test(s)).length / n;62  const longText = strings.filter((s) => s.length > 60 || /\s.*\s/.test(s)).length / n;63  const countLike = strings.filter((s) => /^[\d.,\s]+\s*[kKmMbB]?(\s|$)/.test(s) && /\d/.test(s) && s.length < 40).length / n;6465  // key-name signals (weaker than value signals, but combine)66  if (ID_KEY.test(key)) add("identifier", idLike > 0.5 ? 0.95 : numbers.length / n > 0.5 ? 0.85 : 0.6);67  else if (idLike > 0.8 && strings.length === values.length) add("identifier", 0.55);68  if (USERNAME_KEY.test(key) && strings.length) add(handleRatio > 0.3 ? "username" : "display_name", 0.7);69  if (TITLE_KEY.test(key) && strings.length) add("title", 0.85);70  if (TEXT_KEY.test(key) && strings.length) add("text", longText > 0.3 ? 0.85 : 0.6);71  else if (longText > 0.7 && strings.length === values.length) add("text", 0.5);72  if (URL_KEY.test(key)) add("url", urlRatio > 0.5 ? 0.9 : 0.55);73  if (MEDIA_KEY.test(key) && urlRatio > 0.3) add("media_url", 0.7);74  if (THUMB_KEY.test(key) && urlRatio > 0.3) add("thumbnail_url", 0.8);75  if (TIME_KEY.test(key)) add("timestamp", 0.7);76  if (COUNT_KEY.test(key)) add("count", numbers.length / n > 0.5 || countLike > 0.5 ? 0.9 : 0.5);77  else if (numbers.length / n > 0.8 && numbers.every((x) => Number.isInteger(x) && x >= 0) && epochRatio < 0.5) add("count", 0.4);78  if (DURATION_KEY.test(key)) add("duration", 0.75);79  if (CURSOR_KEY.test(key) && idLike > 0.3) add("cursor", 0.75);80  if (/^[A-Za-z0-9+/=_-]{80,}$/.test(strings[0] ?? "")) add("cursor", 0.6);8182  if (guesses.size === 0) add("unknown", 0.3);83  return [...guesses.entries()]84    .map(([kind, confidence]) => ({ kind, confidence }))85    .sort((a, b) => b.confidence - a.confidence)86    .slice(0, 3);87}8889/** Structural shape signature: sorted keys + value types, arrays collapsed to their first element. Ids/values excluded. */90export function shapeSignature(v: unknown, depth = 0): string {91  if (depth > 6) return "…";92  if (Array.isArray(v)) return `[${v.length ? shapeSignature(v[0], depth + 1) : ""}]`;93  if (v && typeof v === "object") {94    const keys = Object.keys(v as object).sort().slice(0, 40);95    return `{${keys.map((k) => `${k}:${shapeSignature((v as Record<string, unknown>)[k], depth + 1)}`).join(",")}}`;96  }97  return typeOf(v);98}99100interface Collected {101  types: Set<string>;102  values: unknown[];103  seen: number;104}105106/**107 * Profile a JSON payload. Fields are aggregated across all objects sharing a path108 * (array indices collapsed to []), so repeated objects (feed items) surface naturally.109 */110export function profileJson(root: unknown, opts: { maxNodes?: number } = {}): SchemaProfile {111  const maxNodes = opts.maxNodes ?? 20_000;112  const fields = new Map<string, Collected>();113  const arrayObjectCounts = new Map<string, number>(); // path → number of object elements114  let objectCount = 0;115  let nodes = 0;116117  const visit = (v: unknown, pathParts: string[], depth: number) => {118    if (nodes++ > maxNodes || depth > 30) return;119    if (Array.isArray(v)) {120      const objs = v.filter((x) => x && typeof x === "object" && !Array.isArray(x)).length;121      if (objs >= 2) arrayObjectCounts.set(pathParts.join("."), (arrayObjectCounts.get(pathParts.join(".")) ?? 0) + objs);122      for (const item of v.slice(0, 200)) visit(item, [...pathParts, "[]"], depth + 1);123      return;124    }125    if (v && typeof v === "object") {126      objectCount++;127      for (const [k, val] of Object.entries(v as Record<string, unknown>)) {128        const p = [...pathParts, k].join(".");129        const c = fields.get(p) ?? { types: new Set(), values: [], seen: 0 };130        c.types.add(typeOf(val));131        c.seen++;132        if (c.values.length < 25 && (typeof val !== "object" || val === null)) c.values.push(val);133        fields.set(p, c);134        visit(val, [...pathParts, k], depth + 1);135      }136    }137  };138  visit(root, [], 0);139140  const schemaFields: SchemaField[] = [];141  for (const [p, c] of fields) {142    const key = p.split(".").filter((s) => s !== "[]").pop() ?? p;143    const scalarValues = c.values;144    const semantic = scalarValues.length ? guessFieldSemantics(key, scalarValues) : [{ kind: "unknown" as SemanticFieldKind, confidence: 0.2 }];145    schemaFields.push({146      path: p,147      types: [...c.types],148      semantic,149      examples: scalarValues.slice(0, 3).map((x) => String(x).slice(0, 80)),150      frequency: Math.min(1, c.seen / Math.max(1, objectCount)),151    });152  }153154  const repeated = [...arrayObjectCounts.entries()].sort((a, b) => b[1] - a[1]).map(([p]) => p);155  const candidates = detectEntityCandidates(repeated, schemaFields);156157  return {158    shape_hash: shortHash(shapeSignature(root), 16),159    root_type: Array.isArray(root) ? "array" : root && typeof root === "object" ? "object" : "scalar",160    repeated_object_paths: repeated.slice(0, 20),161    fields: schemaFields.slice(0, 400),162    candidate_entity_types: candidates,163    object_count: objectCount,164  };165}166167/** From repeated-object arrays, guess which entity type each array holds by the semantics of its child fields. */168function detectEntityCandidates(repeated: string[], fields: SchemaField[]): SchemaProfile["candidate_entity_types"] {169  const out: SchemaProfile["candidate_entity_types"] = [];170  for (const p of repeated.slice(0, 30)) {171    const prefix = p ? p + ".[]." : "[].";172    const children = fields.filter((f) => f.path.startsWith(prefix));173    if (children.length === 0) continue;174    const has = (kind: SemanticFieldKind, minConf = 0.5) => children.some((f) => f.semantic.some((s) => s.kind === kind && s.confidence >= minConf));175    const keyHas = (re: RegExp) => children.some((f) => re.test(f.path.slice(prefix.length)));176    const id = has("identifier");177    if (!id) continue;178    let type: EntityType | undefined;179    let conf = 0.5;180    if (keyHas(/video|duration|length|watch|views?/i) || has("duration")) {181      type = "video";182      conf = 0.75;183    } else if (keyHas(/channel|subscri|owner|author.*(name|url)|user_?name|handle|screen_name/i) && !has("text") && !has("title")) {184      type = "profile";185      conf = 0.65;186    } else if (keyHas(/comment|repl|parent_id|depth/i) && has("text")) {187      type = "comment";188      conf = 0.7;189    } else if (has("text") || has("title")) {190      type = "post";191      conf = has("count") ? 0.75 : 0.6;192    } else if (keyHas(/community|subreddit|group|page/i)) {193      type = "community";194      conf = 0.55;195    }196    if (type) out.push({ type, confidence: conf, path: p });197  }198  return out.sort((a, b) => b.confidence - a.confidence).slice(0, 10);199}200