import { closeDb, db, migrate, sensors, sources, sql } from "@websensor/db"; import { closeDispatcher, discoverDomain } from "@websensor/connectors"; import { loadRecent } from "./cluster"; import { config, log } from "./config"; import { closeRedis } from "./redis"; import { runDiscovery, syncRegistry } from "./registry"; import { runSensor } from "./pipeline"; import { diffText, evaluateChange } from "@websensor/core"; import { interpretChange, llmAvailable } from "./interpret"; /** * Operator CLI: * tsx src/cli.ts sync — upsert sources/sensors/entities from config/sources.yaml * tsx src/cli.ts discover [sourceId…] — run discovery (+ promote validated endpoints) * tsx src/cli.ts probe — discovery dry-run for a domain (no DB writes) * tsx src/cli.ts run-once — run the full pipeline for one sensor now * tsx src/cli.ts run-due [n] — run up to n due sensors sequentially */ async function main(): Promise { const [cmd, ...args] = process.argv.slice(2); await migrate(config.databaseUrl); switch (cmd) { case "sync": await syncRegistry(); break; case "discover": await syncRegistry(); await runDiscovery({ sourceIds: args.length ? args : undefined }); break; case "probe": { const r = await discoverDomain(args[0]!, { probePages: true }); for (const e of r) console.log(`${e.type.padEnd(10)} ${e.value.toFixed(2)} ${String(e.itemCount ?? "").padStart(5)} ${e.url} (${e.evidence})`); break; } case "run-once": { await loadRecent(); const s = (await db.select().from(sensors).where(sql`id = ${args[0]}`))[0]; if (!s) throw new Error(`sensor ${args[0]} not found`); const src = (await db.select().from(sources).where(sql`id = ${s.sourceId}`))[0]!; const out = await runSensor(s, src); console.log(`${s.id}: ${out}`); break; } case "run-due": { await loadRecent(); const n = Number(args[0] ?? 20); const rows = await db.select().from(sensors).where(sql`enabled and next_check_at <= now()`).orderBy(sql`next_check_at asc`).limit(n); const srcs = new Map((await db.select().from(sources)).map((s) => [s.id, s])); for (const s of rows) { const out = await runSensor(s, srcs.get(s.sourceId)!); console.log(`${s.id.padEnd(50)} ${out}`); } break; } case "relink-entities": { const { relinkMentionedEntities } = await import("./entities"); const r = await relinkMentionedEntities(Number(args[0] ?? 7)); console.log(JSON.stringify(r)); break; } case "prune-blobs": { const { pruneRawSnapshots } = await import("./retention"); console.log(JSON.stringify(await pruneRawSnapshots({ batch: Number(args[0] ?? 5000) }))); break; } case "refresh-clusters": { const { refreshClusterStates } = await import("./cluster"); await refreshClusterStates(); console.log("cluster states refreshed"); break; } case "llm-test": { if (!llmAvailable()) throw new Error("ANTHROPIC_API_KEY not set"); const before = "API Pricing\nInput: $10 / million tokens\nOutput: $30 / million tokens\nBatch API: 50% discount"; const after = "API Pricing\nInput: $8 / million tokens\nOutput: $24 / million tokens\nBatch API: 50% discount\nPrompt caching: 90% discount on cached input"; const diff = diffText(before, after); const heuristic = evaluateChange(diff, { sensorType: "HTML", url: "https://example-ai.com/pricing", sourceCategories: ["ai"], title: "API Pricing" }); const t0 = Date.now(); const out = await interpretChange({ sourceName: "Example AI", sourceCategories: ["ai"], url: "https://example-ai.com/pricing", sensorName: "pricing", sensorType: "HTML", heuristic, diff, prelimImportance: Number(args[0] ?? 60), title: "API Pricing" }); console.log(JSON.stringify(out, null, 2), `\n${Date.now() - t0} ms`); break; } case "expand": { // Deep discovery dry-run (no DB writes): what the Source Factory would find for a domain. const { discoverOrganization } = await import("@websensor/connectors"); const { scoreCandidate } = await import("./factory/score"); const hintsArg = args.find((a) => a.startsWith("--hints=")); const hints = hintsArg ? (JSON.parse(hintsArg.slice(8)) as Record) : {}; const r = await discoverOrganization(args[0]!, { hints, probePosture: args.includes("--posture") }); console.log(`${r.domain} → origin ${r.origin} · home ${r.homeStatus}${r.blocked ? " BLOCKED" : ""} · ${r.requests} requests · ${r.durationMs} ms`); for (const n of r.notes) console.log(` note: ${n}`); for (const c of r.candidates) { const s = scoreCandidate(c, { importance: Number(args.find((a) => a.startsWith("--importance="))?.slice(13) ?? 2) }); console.log(`${s.score.toFixed(2)} ${c.kind.padEnd(10)} ${c.connector.padEnd(10)} ${c.suggestedTier} ${String(c.itemCount ?? "").padStart(4)} ${c.url} (${c.evidence})`); } if (r.rejected.length) console.log(`rejected: ${r.rejected.map((x) => `${x.url} [${x.reason}]`).join("\n ")}`); break; } case "factory": { const f = await import("./factory"); const sub = args[0]; const flagVal = (n: string): string | undefined => args.find((a) => a.startsWith(`--${n}=`))?.slice(n.length + 3); if (sub === "seed") { const files = await f.seedFromFiles(); const mode = (flagVal("mode") ?? "hinted") as "uncovered" | "hinted" | "all"; const cov = await f.seedFromCoverage({ mode, sectors: flagVal("sector")?.split(","), requeue: args.includes("--requeue") }); console.log(JSON.stringify({ files, coverage: cov }, null, 2)); } else if (sub === "run") { const n = Number(flagVal("n") ?? 6); const ids = args.slice(1).filter((a) => !a.startsWith("--")); const out = await f.runFactoryBatch(n, ids.length ? { seedIds: ids } : {}); for (const o of out) console.log(`${o.status.padEnd(10)} ${o.seedId.padEnd(40)} source=${o.sourceId ?? "-"} candidates=${o.candidates} shadow=${o.shadow} requests=${o.requests} ${o.durationMs} ms${o.notes.length ? " · " + o.notes.join(" · ") : ""}`); } else if (sub === "evaluate") { const r = await f.evaluateShadows({ force: args.includes("--force") }); for (const d of r.decisions.filter((x) => x.decision !== "deferred" || args.includes("--verbose"))) console.log(`${d.decision.padEnd(9)} ${d.sensorId.padEnd(60)} ${d.reason}`); console.log(JSON.stringify({ accepted: r.accepted, rejected: r.rejected, deferred: r.deferred })); } else if (sub === "stats") { console.log(JSON.stringify(await f.factoryStats(), null, 2)); } else if (sub === "export") { process.stdout.write(await f.exportFactoryFragment({ sector: flagVal("sector"), sinceDays: flagVal("since") ? Number(flagVal("since")) : undefined, includeShadow: args.includes("--include-shadow") })); } else if (sub === "requeue") { const ids = args.slice(1).filter((a) => !a.startsWith("--")); const r = await db.execute(sql`update factory_seeds set status = 'queued', updated_at = now() where ${ids.length ? sql`id = any(${sql.raw("array[" + ids.map((i) => "'" + i.replace(/'/g, "''") + "'").join(",") + "]::text[]")})` : sql`status in ('error','blocked')`}`); console.log(`requeued ${r.rowCount ?? 0}`); } else { console.error("usage: cli.ts factory seed [--mode=uncovered|hinted|all] [--sector=a,b] [--requeue] | run [seedId…] [--n=6] | evaluate [--force] [--verbose] | stats | export [--sector=x] [--since=days] [--include-shadow] | requeue [seedId…]"); process.exitCode = 1; } break; } default: console.error("usage: cli.ts sync | discover [sourceId…] | probe | expand [--hints=json] [--posture] | run-once | run-due [n] | relink-entities [days] | prune-blobs [batch] | refresh-clusters | factory "); process.exitCode = 1; } } main() .catch((e) => { log.error({ err: (e as Error).stack ?? String(e) }, "cli failed"); process.exitCode = 1; }) .finally(async () => { await closeDispatcher(); await closeRedis(); await closeDb(); });