spb/social-runtime-crawler

1import fs from "node:fs";2import path from "node:path";3import { DEFAULT_BUDGET, isPlatform, loadConfig, newId, type AgentMode, type CrawlJob, type MediaLevel, type Platform } from "@src/shared";4import { interactiveLogin } from "@src/browser";5import { JsonlEventLog } from "@src/events";6import { PlatformModel, compileConnector } from "@src/platform-model";7import { CrawlEngine } from "./engine.ts";89/**10 * social-runtime CLI (§74):11 *   pnpm src login <platform> [--account alias]12 *   pnpm src crawl <platform> --goal "…" [--mode research|observe|topic|profile|learn] [--query "…"] [--seed url] [--minutes 10] [--actions 60] [--media 2] [--headless]13 *   pnpm src learn <platform> [--minutes 15]          (PLATFORM LEARNING MODE + connector compilation)14 *   pnpm src replay <session_id> [--type EVENT_TYPE]  (inspect why the crawler did what it did)15 *   pnpm src status <platform>                        (learned model summary)16 */17const [, , cmd, ...rest] = process.argv;18const args = parseArgs(rest);19const cfg = loadConfig();2021function parseArgs(list: string[]): { positional: string[]; flags: Record<string, string | boolean> } {22  const positional: string[] = [];23  const flags: Record<string, string | boolean> = {};24  for (let i = 0; i < list.length; i++) {25    const a = list[i]!;26    if (a.startsWith("--")) {27      const key = a.slice(2);28      const next = list[i + 1];29      if (next !== undefined && !next.startsWith("--")) {30        flags[key] = next;31        i++;32      } else flags[key] = true;33    } else positional.push(a);34  }35  return { positional, flags };36}3738function platformArg(): Platform {39  const p = args.positional[0];40  if (!p || !isPlatform(p)) {41    console.error(`Usage: ${cmd} <platform> — one of youtube, reddit, facebook, instagram, tiktok, x, linkedin, threads`);42    process.exit(2);43  }44  return p;45}4647function str(k: string, d?: string): string | undefined {48  const v = args.flags[k];49  return typeof v === "string" ? v : d;50}51function num(k: string, d: number): number {52  const v = args.flags[k];53  return typeof v === "string" && !Number.isNaN(Number(v)) ? Number(v) : d;54}5556async function main() {57  switch (cmd) {58    case "login": {59      const platform = platformArg();60      await interactiveLogin({ platform, accountAlias: str("account", "research-01")!, profilesDir: cfg.profilesDir, channel: cfg.browserChannel });61      return;62    }63    case "crawl":64    case "learn": {65      const platform = platformArg();66      const isLearn = cmd === "learn";67      const mode = (str("mode", isLearn ? "learn" : "research") as AgentMode) ?? "research";68      const query = str("query") ?? str("q");69      const goal = str("goal") ?? (isLearn ? `Learn how ${platform} surfaces content: page types, entity structures, response schemas, navigation` : query ? `Discover public content about “${query}”` : `Observe the ${platform} feed`);70      if (args.flags.headless) cfg.headless = true;71      const job: CrawlJob = {72        job_id: str("job") ?? newId("job"),73        platform,74        account_alias: str("account", "research-01")!,75        mode,76        goal,77        query,78        seed_url: str("seed"),79        budget: { ...DEFAULT_BUDGET, max_minutes: num("minutes", isLearn ? 15 : DEFAULT_BUDGET.max_minutes), max_actions: num("actions", isLearn ? 80 : DEFAULT_BUDGET.max_actions), max_profiles: num("profiles", DEFAULT_BUDGET.max_profiles), max_posts: num("posts", DEFAULT_BUDGET.max_posts), max_videos: num("videos", DEFAULT_BUDGET.max_videos), max_depth: num("depth", DEFAULT_BUDGET.max_depth) },80        media_level: Math.max(0, Math.min(4, num("media", 1))) as MediaLevel,81        stay_on_platform: true,82        read_only: true,83      };84      const engine = new CrawlEngine(cfg, job);85      const summary = await engine.run();86      printSummary(summary);87      return;88    }89    case "status": {90      const platform = platformArg();91      const model = new PlatformModel(platform, cfg.platformModelDir);92      const s = model.summary();93      console.log(renderLearned(s));94      if (args.flags.compile) console.log("compiled →", compileConnector(model, path.join(process.cwd(), "connectors", platform)));95      return;96    }97    case "replay": {98      const id = args.positional[0];99      if (!id) {100        const dirs = fs.existsSync(cfg.sessionsDir) ? fs.readdirSync(cfg.sessionsDir).sort() : [];101        console.log("sessions:\n" + dirs.map((d) => "  " + d).join("\n"));102        return;103      }104      const events = JsonlEventLog.read(path.join(cfg.sessionsDir, id));105      const type = str("type");106      for (const ev of events) {107        if (type && ev.event_type !== type) continue;108        if (["ACTION_PLANNED", "ACTION_EXECUTED", "ACTION_FAILED", "PAGE_OPENED", "NETWORK_SCHEMA_DISCOVERED", "CONNECTOR_PATTERN_LEARNED", "CONNECTOR_DEGRADED", "LOOP_DETECTED", "AUTH_REQUIRED", "BUDGET_EXHAUSTED", "MEDIA_DISCOVERED"].includes(ev.event_type) || type) {109          const p = ev.payload as Record<string, unknown>;110          const brief = ev.event_type === "ACTION_PLANNED" ? `${(p.action as { type: string; label: string }).type} — ${(p.action as { label: string }).label} | gain=${Number(p.expected_information_gain).toFixed(3)} | ${p.planner} | ${p.reason}` : ev.event_type === "PAGE_OPENED" ? `${p.page_type} (${Number(p.confidence ?? 0).toFixed(2)}) ${p.url} · entities=${p.entities} dom=${p.dom_entities} net=${p.network_entities} both=${p.both_surfaces}` : ev.event_type === "NETWORK_SCHEMA_DISCOVERED" ? `${(p.fingerprint as { hostname: string; path_pattern: string }).hostname}${(p.fingerprint as { path_pattern: string }).path_pattern} → ${JSON.stringify((p.schema as { candidate_entity_types: unknown }).candidate_entity_types)}` : JSON.stringify(p).slice(0, 220);111          console.log(`${ev.timestamp.slice(11, 19)} #${ev.step ?? 0} ${ev.event_type.padEnd(26)} ${brief}`);112        }113      }114      return;115    }116    default:117      console.log(`social-runtime — Social Runtime Crawler118119  pnpm src login <platform> [--account alias]120  pnpm src crawl <platform> --goal "…" [--query "…"] [--mode research|observe|topic|profile|learn] [--seed url] [--minutes 10] [--actions 60] [--media 0-4] [--headless]121  pnpm src learn <platform> [--minutes 15]122  pnpm src status <platform> [--compile]123  pnpm src replay [session_id] [--type EVENT_TYPE]124  pnpm dashboard        → http://localhost:${cfg.dashboardPort}125`);126  }127}128129function printSummary(s: Awaited<ReturnType<CrawlEngine["run"]>>) {130  console.log(`131Session ${s.session_id} (${s.platform}) — ended: ${s.ended_because}132  steps: ${s.steps}   entities: ${s.entities}   videos: ${s.videos}133  network responses: ${s.network_responses}   distinct schemas: ${s.schemas}   patterns learned: ${s.patterns_learned}134  world: ${JSON.stringify(s.world.by_type)}  multi-surface: ${s.world.multi_surface}135${renderLearned(s.connector)}136  log: ${s.session_dir}/events.jsonl137`);138}139140function renderLearned(s: ReturnType<PlatformModel["summary"]>): string {141  return `Platform ${s.platform} learned (${s.sessions} session${s.sessions === 1 ? "" : "s"})142  Page types: ${s.page_types}143  Entity types: ${s.entity_types}144  Navigation actions: ${s.navigation_actions}145  Network schemas: ${s.network_schemas}146  Media patterns: ${s.media_patterns}147  Confidence: ${s.confidence}%`;148}149150main().catch((err) => {151  console.error(err);152  process.exit(1);153});154