import fs from "node:fs"; import path from "node:path"; import { DEFAULT_BUDGET, isPlatform, loadConfig, newId, type AgentMode, type CrawlJob, type MediaLevel, type Platform } from "@src/shared"; import { interactiveLogin } from "@src/browser"; import { JsonlEventLog } from "@src/events"; import { PlatformModel, compileConnector } from "@src/platform-model"; import { CrawlEngine } from "./engine.ts"; /** * social-runtime CLI (§74): * pnpm src login [--account alias] * pnpm src crawl --goal "…" [--mode research|observe|topic|profile|learn] [--query "…"] [--seed url] [--minutes 10] [--actions 60] [--media 2] [--headless] * pnpm src learn [--minutes 15] (PLATFORM LEARNING MODE + connector compilation) * pnpm src replay [--type EVENT_TYPE] (inspect why the crawler did what it did) * pnpm src status (learned model summary) */ const [, , cmd, ...rest] = process.argv; const args = parseArgs(rest); const cfg = loadConfig(); function parseArgs(list: string[]): { positional: string[]; flags: Record } { const positional: string[] = []; const flags: Record = {}; for (let i = 0; i < list.length; i++) { const a = list[i]!; if (a.startsWith("--")) { const key = a.slice(2); const next = list[i + 1]; if (next !== undefined && !next.startsWith("--")) { flags[key] = next; i++; } else flags[key] = true; } else positional.push(a); } return { positional, flags }; } function platformArg(): Platform { const p = args.positional[0]; if (!p || !isPlatform(p)) { console.error(`Usage: ${cmd} — one of youtube, reddit, facebook, instagram, tiktok, x, linkedin, threads`); process.exit(2); } return p; } function str(k: string, d?: string): string | undefined { const v = args.flags[k]; return typeof v === "string" ? v : d; } function num(k: string, d: number): number { const v = args.flags[k]; return typeof v === "string" && !Number.isNaN(Number(v)) ? Number(v) : d; } async function main() { switch (cmd) { case "login": { const platform = platformArg(); await interactiveLogin({ platform, accountAlias: str("account", "research-01")!, profilesDir: cfg.profilesDir, channel: cfg.browserChannel }); return; } case "crawl": case "learn": { const platform = platformArg(); const isLearn = cmd === "learn"; const mode = (str("mode", isLearn ? "learn" : "research") as AgentMode) ?? "research"; const query = str("query") ?? str("q"); const goal = str("goal") ?? (isLearn ? `Learn how ${platform} surfaces content: page types, entity structures, response schemas, navigation` : query ? `Discover public content about “${query}”` : `Observe the ${platform} feed`); if (args.flags.headless) cfg.headless = true; const job: CrawlJob = { job_id: str("job") ?? newId("job"), platform, account_alias: str("account", "research-01")!, mode, goal, query, seed_url: str("seed"), budget: { ...DEFAULT_BUDGET, max_minutes: num("minutes", isLearn ? 15 : DEFAULT_BUDGET.max_minutes), max_actions: num("actions", isLearn ? 80 : DEFAULT_BUDGET.max_actions), max_profiles: num("profiles", DEFAULT_BUDGET.max_profiles), max_posts: num("posts", DEFAULT_BUDGET.max_posts), max_videos: num("videos", DEFAULT_BUDGET.max_videos), max_depth: num("depth", DEFAULT_BUDGET.max_depth) }, media_level: Math.max(0, Math.min(4, num("media", 1))) as MediaLevel, stay_on_platform: true, read_only: true, }; const engine = new CrawlEngine(cfg, job); const summary = await engine.run(); printSummary(summary); return; } case "status": { const platform = platformArg(); const model = new PlatformModel(platform, cfg.platformModelDir); const s = model.summary(); console.log(renderLearned(s)); if (args.flags.compile) console.log("compiled →", compileConnector(model, path.join(process.cwd(), "connectors", platform))); return; } case "replay": { const id = args.positional[0]; if (!id) { const dirs = fs.existsSync(cfg.sessionsDir) ? fs.readdirSync(cfg.sessionsDir).sort() : []; console.log("sessions:\n" + dirs.map((d) => " " + d).join("\n")); return; } const events = JsonlEventLog.read(path.join(cfg.sessionsDir, id)); const type = str("type"); for (const ev of events) { if (type && ev.event_type !== type) continue; if (["ACTION_PLANNED", "ACTION_EXECUTED", "ACTION_FAILED", "PAGE_OPENED", "NETWORK_SCHEMA_DISCOVERED", "CONNECTOR_PATTERN_LEARNED", "CONNECTOR_DEGRADED", "LOOP_DETECTED", "AUTH_REQUIRED", "BUDGET_EXHAUSTED", "MEDIA_DISCOVERED"].includes(ev.event_type) || type) { const p = ev.payload as Record; const brief = ev.event_type === "ACTION_PLANNED" ? `${(p.action as { type: string; label: string }).type} — ${(p.action as { label: string }).label} | gain=${Number(p.expected_information_gain).toFixed(3)} | ${p.planner} | ${p.reason}` : ev.event_type === "PAGE_OPENED" ? `${p.page_type} (${Number(p.confidence ?? 0).toFixed(2)}) ${p.url} · entities=${p.entities} dom=${p.dom_entities} net=${p.network_entities} both=${p.both_surfaces}` : ev.event_type === "NETWORK_SCHEMA_DISCOVERED" ? `${(p.fingerprint as { hostname: string; path_pattern: string }).hostname}${(p.fingerprint as { path_pattern: string }).path_pattern} → ${JSON.stringify((p.schema as { candidate_entity_types: unknown }).candidate_entity_types)}` : JSON.stringify(p).slice(0, 220); console.log(`${ev.timestamp.slice(11, 19)} #${ev.step ?? 0} ${ev.event_type.padEnd(26)} ${brief}`); } } return; } default: console.log(`social-runtime — Social Runtime Crawler pnpm src login [--account alias] pnpm src crawl --goal "…" [--query "…"] [--mode research|observe|topic|profile|learn] [--seed url] [--minutes 10] [--actions 60] [--media 0-4] [--headless] pnpm src learn [--minutes 15] pnpm src status [--compile] pnpm src replay [session_id] [--type EVENT_TYPE] pnpm dashboard → http://localhost:${cfg.dashboardPort} `); } } function printSummary(s: Awaited>) { console.log(` Session ${s.session_id} (${s.platform}) — ended: ${s.ended_because} steps: ${s.steps} entities: ${s.entities} videos: ${s.videos} network responses: ${s.network_responses} distinct schemas: ${s.schemas} patterns learned: ${s.patterns_learned} world: ${JSON.stringify(s.world.by_type)} multi-surface: ${s.world.multi_surface} ${renderLearned(s.connector)} log: ${s.session_dir}/events.jsonl `); } function renderLearned(s: ReturnType): string { return `Platform ${s.platform} learned (${s.sessions} session${s.sessions === 1 ? "" : "s"}) Page types: ${s.page_types} Entity types: ${s.entity_types} Navigation actions: ${s.navigation_actions} Network schemas: ${s.network_schemas} Media patterns: ${s.media_patterns} Confidence: ${s.confidence}%`; } main().catch((err) => { console.error(err); process.exit(1); });