TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { closeDb, db, migrate, sensors, sources, sql } from "@websensor/db";2import { closeDispatcher, discoverDomain } from "@websensor/connectors";3import { loadRecent } from "./cluster";4import { config, log } from "./config";5import { closeRedis } from "./redis";6import { runDiscovery, syncRegistry } from "./registry";7import { runSensor } from "./pipeline";8import { diffText, evaluateChange } from "@websensor/core";9import { interpretChange, llmAvailable } from "./interpret";1011/**12 * Operator CLI:13 * tsx src/cli.ts sync — upsert sources/sensors/entities from config/sources.yaml14 * tsx src/cli.ts discover [sourceId…] — run discovery (+ promote validated endpoints)15 * tsx src/cli.ts probe <domain> — discovery dry-run for a domain (no DB writes)16 * tsx src/cli.ts run-once <sensorId> — run the full pipeline for one sensor now17 * tsx src/cli.ts run-due [n] — run up to n due sensors sequentially18 */19async function main(): Promise<void> {20 const [cmd, ...args] = process.argv.slice(2);21 await migrate(config.databaseUrl);22 switch (cmd) {23 case "sync":24 await syncRegistry();25 break;26 case "discover":27 await syncRegistry();28 await runDiscovery({ sourceIds: args.length ? args : undefined });29 break;30 case "probe": {31 const r = await discoverDomain(args[0]!, { probePages: true });32 for (const e of r) console.log(`${e.type.padEnd(10)} ${e.value.toFixed(2)} ${String(e.itemCount ?? "").padStart(5)} ${e.url} (${e.evidence})`);33 break;34 }35 case "run-once": {36 await loadRecent();37 const s = (await db.select().from(sensors).where(sql`id = ${args[0]}`))[0];38 if (!s) throw new Error(`sensor ${args[0]} not found`);39 const src = (await db.select().from(sources).where(sql`id = ${s.sourceId}`))[0]!;40 const out = await runSensor(s, src);41 console.log(`${s.id}: ${out}`);42 break;43 }44 case "run-due": {45 await loadRecent();46 const n = Number(args[0] ?? 20);47 const rows = await db.select().from(sensors).where(sql`enabled and next_check_at <= now()`).orderBy(sql`next_check_at asc`).limit(n);48 const srcs = new Map((await db.select().from(sources)).map((s) => [s.id, s]));49 for (const s of rows) {50 const out = await runSensor(s, srcs.get(s.sourceId)!);51 console.log(`${s.id.padEnd(50)} ${out}`);52 }53 break;54 }55 case "relink-entities": {56 const { relinkMentionedEntities } = await import("./entities");57 const r = await relinkMentionedEntities(Number(args[0] ?? 7));58 console.log(JSON.stringify(r));59 break;60 }61 case "prune-blobs": {62 const { pruneRawSnapshots } = await import("./retention");63 console.log(JSON.stringify(await pruneRawSnapshots({ batch: Number(args[0] ?? 5000) })));64 break;65 }66 case "refresh-clusters": {67 const { refreshClusterStates } = await import("./cluster");68 await refreshClusterStates();69 console.log("cluster states refreshed");70 break;71 }72 case "llm-test": {73 if (!llmAvailable()) throw new Error("ANTHROPIC_API_KEY not set");74 const before = "API Pricing\nInput: $10 / million tokens\nOutput: $30 / million tokens\nBatch API: 50% discount";75 const after = "API Pricing\nInput: $8 / million tokens\nOutput: $24 / million tokens\nBatch API: 50% discount\nPrompt caching: 90% discount on cached input";76 const diff = diffText(before, after);77 const heuristic = evaluateChange(diff, { sensorType: "HTML", url: "https://example-ai.com/pricing", sourceCategories: ["ai"], title: "API Pricing" });78 const t0 = Date.now();79 const out = await interpretChange({ sourceName: "Example AI", sourceCategories: ["ai"], url: "https://example-ai.com/pricing", sensorName: "pricing", sensorType: "HTML", heuristic, diff, prelimImportance: Number(args[0] ?? 60), title: "API Pricing" });80 console.log(JSON.stringify(out, null, 2), `\n${Date.now() - t0} ms`);81 break;82 }83 case "expand": {84 // Deep discovery dry-run (no DB writes): what the Source Factory would find for a domain.85 const { discoverOrganization } = await import("@websensor/connectors");86 const { scoreCandidate } = await import("./factory/score");87 const hintsArg = args.find((a) => a.startsWith("--hints="));88 const hints = hintsArg ? (JSON.parse(hintsArg.slice(8)) as Record<string, unknown>) : {};89 const r = await discoverOrganization(args[0]!, { hints, probePosture: args.includes("--posture") });90 console.log(`${r.domain} → origin ${r.origin} · home ${r.homeStatus}${r.blocked ? " BLOCKED" : ""} · ${r.requests} requests · ${r.durationMs} ms`);91 for (const n of r.notes) console.log(` note: ${n}`);92 for (const c of r.candidates) {93 const s = scoreCandidate(c, { importance: Number(args.find((a) => a.startsWith("--importance="))?.slice(13) ?? 2) });94 console.log(`${s.score.toFixed(2)} ${c.kind.padEnd(10)} ${c.connector.padEnd(10)} ${c.suggestedTier} ${String(c.itemCount ?? "").padStart(4)} ${c.url} (${c.evidence})`);95 }96 if (r.rejected.length) console.log(`rejected: ${r.rejected.map((x) => `${x.url} [${x.reason}]`).join("\n ")}`);97 break;98 }99 case "factory": {100 const f = await import("./factory");101 const sub = args[0];102 const flagVal = (n: string): string | undefined => args.find((a) => a.startsWith(`--${n}=`))?.slice(n.length + 3);103 if (sub === "seed") {104 const files = await f.seedFromFiles();105 const mode = (flagVal("mode") ?? "hinted") as "uncovered" | "hinted" | "all";106 const cov = await f.seedFromCoverage({ mode, sectors: flagVal("sector")?.split(","), requeue: args.includes("--requeue") });107 console.log(JSON.stringify({ files, coverage: cov }, null, 2));108 } else if (sub === "run") {109 const n = Number(flagVal("n") ?? 6);110 const ids = args.slice(1).filter((a) => !a.startsWith("--"));111 const out = await f.runFactoryBatch(n, ids.length ? { seedIds: ids } : {});112 for (const o of out) console.log(`${o.status.padEnd(10)} ${o.seedId.padEnd(40)} source=${o.sourceId ?? "-"} candidates=${o.candidates} shadow=${o.shadow} requests=${o.requests} ${o.durationMs} ms${o.notes.length ? " · " + o.notes.join(" · ") : ""}`);113 } else if (sub === "evaluate") {114 const r = await f.evaluateShadows({ force: args.includes("--force") });115 for (const d of r.decisions.filter((x) => x.decision !== "deferred" || args.includes("--verbose"))) console.log(`${d.decision.padEnd(9)} ${d.sensorId.padEnd(60)} ${d.reason}`);116 console.log(JSON.stringify({ accepted: r.accepted, rejected: r.rejected, deferred: r.deferred }));117 } else if (sub === "stats") {118 console.log(JSON.stringify(await f.factoryStats(), null, 2));119 } else if (sub === "export") {120 process.stdout.write(await f.exportFactoryFragment({ sector: flagVal("sector"), sinceDays: flagVal("since") ? Number(flagVal("since")) : undefined, includeShadow: args.includes("--include-shadow") }));121 } else if (sub === "requeue") {122 const ids = args.slice(1).filter((a) => !a.startsWith("--"));123 const r = await db.execute(sql`update factory_seeds set status = 'queued', updated_at = now() where ${ids.length ? sql`id = any(${sql.raw("array[" + ids.map((i) => "'" + i.replace(/'/g, "''") + "'").join(",") + "]::text[]")})` : sql`status in ('error','blocked')`}`);124 console.log(`requeued ${r.rowCount ?? 0}`);125 } else {126 console.error("usage: cli.ts factory seed [--mode=uncovered|hinted|all] [--sector=a,b] [--requeue] | run [seedId…] [--n=6] | evaluate [--force] [--verbose] | stats | export [--sector=x] [--since=days] [--include-shadow] | requeue [seedId…]");127 process.exitCode = 1;128 }129 break;130 }131 default:132 console.error("usage: cli.ts sync | discover [sourceId…] | probe <domain> | expand <domain> [--hints=json] [--posture] | run-once <sensorId> | run-due [n] | relink-entities [days] | prune-blobs [batch] | refresh-clusters | factory <seed|run|evaluate|stats|export|requeue>");133 process.exitCode = 1;134 }135}136137main()138 .catch((e) => {139 log.error({ err: (e as Error).stack ?? String(e) }, "cli failed");140 process.exitCode = 1;141 })142 .finally(async () => {143 await closeDispatcher();144 await closeRedis();145 await closeDb();146 });147