spb/rareindex

1/**2 * Live smoke / fixture capture for this wave's connectors (works before the registry is rebuilt).3 *   pnpm tsx connectors/api/_wlib/smoke.ts api/reverb [--limit 2] [--save <fixtureName>] [--trim 8] [--mode probe|incremental]4 */5import path from 'node:path';6import { pathToFileURL } from 'node:url';7import { readFileSync } from 'node:fs';8import { createRouter, createCrawlContext, ConnectorMetaSchema, type RareIndexConnector, type ConnectorMeta } from '@rareindex/connectors';9import { saveFixture } from '@rareindex/connectors/testing';1011const [, , mod, ...rest] = process.argv;12if (!mod) throw new Error('usage: smoke.ts <engine>/<id> [--limit N] [--save name] [--trim N]');13const arg = (k: string) => {14  const i = rest.indexOf(k);15  return i >= 0 ? rest[i + 1] : undefined;16};17const limit = Number(arg('--limit') ?? 2);18const save = arg('--save');19const trim = Number(arg('--trim') ?? 8);20const mode = (arg('--mode') ?? 'probe') as 'probe' | 'incremental' | 'backfill';2122const dir = path.resolve(process.cwd(), 'connectors', mod);23const meta: ConnectorMeta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));24const factory = (await import(pathToFileURL(path.join(dir, 'index.ts')).href)) as { default: (m: ConnectorMeta) => RareIndexConnector };25const connector = factory.default(meta);26const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });27const ctx = createCrawlContext({ router, meta, options: { mode, limit } });2829let i = 0;30for await (const raw of connector.crawl(ctx)) {31  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });32  console.log(`raw ${raw.externalId} (${raw.kind}) → ${out.length} records`);33  for (const r of out.slice(0, 3)) console.log('  ', JSON.stringify(r).slice(0, 420));34  if (save && i === 0) {35    const p = raw.payload as Record<string, unknown>;36    for (const k of ['items', 'listings', 'products', 'results', 'lots']) if (Array.isArray(p[k])) p[k] = (p[k] as unknown[]).slice(0, trim);37    saveFixture(meta.id, save, {38      raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p },39      expect: { minCount: 1 },40      note: `Captured live from ${new URL(raw.url).host} on ${new Date().toISOString().slice(0, 10)} (trimmed to ${trim} items)`,41    });42    console.log(`saved fixture data/fixtures/${meta.id}/${save}.json`);43  }44  i++;45}46console.log('engineStats', JSON.stringify(ctx.engineStats), 'anomalies', ctx.anomalies);47