spb/datacenterindex

1/**2 * Dry-run a connector against LIVE sources without persisting anything.3 *   pnpm tsx scripts/try-connector.ts config/connectors/<id>.yaml [--limit 5] [--url <single url>] [--verbose] [--json out.json]4 * Registers every apps/worker/src/connectors/<group>/index.ts (export function register()) found on disk.5 */6import { readFileSync, readdirSync, existsSync } from "node:fs";7import { join, resolve } from "node:path";8import { parseConnectorConfig, GenericConnector, getImplementation, createTestContext, type Connector, type DiscoveredUrl } from "@dci/connectors";910const args = process.argv.slice(2);11const file = args.find((a) => !a.startsWith("--"));12if (!file) { console.error("usage: try-connector.ts <config.yaml> [--limit N] [--url U] [--verbose] [--json out]"); process.exit(2); }13const opt = (k: string) => { const i = args.indexOf(`--${k}`); return i >= 0 ? args[i + 1] : undefined; };14const limit = Number(opt("limit") ?? 5);15const only = opt("url");16const verbose = args.includes("--verbose");17const jsonOut = opt("json");1819async function registerAll() {20  const dir = resolve("apps/worker/src/connectors");21  if (!existsSync(dir)) return;22  for (const g of readdirSync(dir, { withFileTypes: true })) {23    if (!g.isDirectory()) continue;24    const idx = join(dir, g.name, "index.ts");25    if (!existsSync(idx)) continue;26    const mod = (await import(idx)) as { register?: () => void };27    mod.register?.();28  }29}3031async function main() {32  await registerAll();33  const cfg = parseConnectorConfig(readFileSync(file!, "utf8"), file);34  const impl = cfg.implementation ? getImplementation(cfg.implementation) : undefined;35  if (cfg.implementation && !impl) throw new Error(`implementation "${cfg.implementation}" not registered`);36  const connector: Connector = impl ? impl(cfg) : new GenericConnector(cfg);37  const ctx = createTestContext(cfg, { verbose });38  const t0 = Date.now();39  let urls: DiscoveredUrl[] = only ? [{ url: only, group: "manual", priority: 100 }] : await connector.discover(ctx);40  console.error(`discovered ${urls.length} urls in ${Date.now() - t0}ms`);41  const groups = new Map<string, number>();42  for (const u of urls) groups.set(u.group, (groups.get(u.group) ?? 0) + 1);43  console.error(`groups: ${[...groups.entries()].map(([g, n]) => `${g}=${n}`).join(", ")}`);44  urls = urls.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)).slice(0, limit);45  const all: unknown[] = [];46  let totalValid = 0, totalRejected = 0;47  for (const u of urls) {48    const doc = await connector.fetch(ctx, u);49    if (doc.error) { console.error(`✗ ${u.url} → ${doc.error.code}: ${doc.error.message}`); continue; }50    const recs = await connector.extract(ctx, doc);51    const ents = await connector.normalize(ctx, recs);52    const report = await connector.validate(ctx, ents);53    totalValid += report.valid; totalRejected += report.rejected;54    console.error(`✓ ${u.url} [${doc.fetcher} L${doc.level} ${doc.status} ${doc.durationMs}ms] → ${recs.length} records, ${ents.length} entities, ${report.valid} valid, ${report.rejected} rejected${(doc.meta?.pageType as string | undefined) ? " · " + doc.meta!.pageType : ""}`);55    for (const i of report.issues) console.error(`   ${i.level === "error" ? "ERR " : "warn"} ${i.key} ${i.field ?? ""}: ${i.message}`);56    for (const e of ents.slice(0, 3)) console.log(JSON.stringify(e, null, verbose ? 1 : 0).slice(0, verbose ? 4000 : 700));57    all.push(...ents);58  }59  console.error(`\nTOTAL entities=${all.length} valid=${totalValid} rejected=${totalRejected} credits=${ctx.credits} fetched=${ctx.fetched.length}`);60  if (jsonOut) { const { writeFileSync } = await import("node:fs"); writeFileSync(jsonOut, JSON.stringify(all, null, 1)); console.error(`wrote ${jsonOut}`); }61  process.exit(totalRejected > 0 && totalValid === 0 ? 1 : 0);62}63main().catch((e) => { console.error(e); process.exit(1); });64