spb/rareindex

1/**2 * Shared live smoke/capture runner for the auction-house connectors. Loads meta.json + index.ts from3 * the connector folder directly (works before the registry is rebuilt).4 * Usage: set -a; . ./.env; set +a; pnpm tsx connectors/<engine>/<id>/_smoke.ts [limit]5 */6import { readFileSync } from 'node:fs';7import path from 'node:path';8import { ConnectorMetaSchema, createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors';9import { saveFixture } from '@rareindex/connectors/testing';10import { childLogger } from '@rareindex/shared';1112export async function loadLocal(dir: string): Promise<{ meta: ConnectorMeta; connector: RareIndexConnector }> {13  const raw = JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'));14  const meta = ConnectorMetaSchema.parse(raw);15  const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector };16  return { meta, connector: mod.default(meta) };17}1819export async function runSmoke(dir: string, limit = Number(process.argv[2] ?? 2)): Promise<void> {20  const { meta, connector } = await loadLocal(dir);21  const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });22  const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: meta.id, smoke: true }) });23  let raws = 0;24  let total = 0;25  const samples: unknown[] = [];26  for await (const raw of connector.crawl(ctx)) {27    raws++;28    const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });29    total += records.length;30    console.log(`raw ${raw.url} → ${records.length} records`);31    for (const r of records.slice(0, 3)) samples.push(r);32    if (raws >= limit) break;33  }34  for (const s of samples.slice(0, 3)) console.log(JSON.stringify(s, null, 1));35  console.log(JSON.stringify({ raws, totalNormalized: total, engineStats: ctx.engineStats, anomalies: ctx.anomalies }));36}3738/** Capture the first probe raw record as a fixture (item lists trimmed to `keep`). */39export async function captureFixture(dir: string, name: string, keep = 4): Promise<void> {40  const { meta, connector } = await loadLocal(dir);41  const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });42  const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 }, log: childLogger({ connector: meta.id, capture: true }) });43  for await (const raw of connector.crawl(ctx)) {44    const payload = raw.payload as Record<string, unknown>;45    for (const k of ['items', 'lots', 'cards', 'listings']) {46      if (Array.isArray(payload[k])) payload[k] = (payload[k] as unknown[]).slice(0, keep);47    }48    const fetchedAt = raw.fetchedAt ?? new Date();49    const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt });50    saveFixture(meta.id, name, {51      raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt, payload },52      expect: { count: records.length, kinds: [...new Set(records.map((r) => r.kind))] },53      note: `Captured live from ${raw.url} (lists trimmed to ${keep}).`,54    });55    console.log(`saved data/fixtures/${meta.id}/${name}.json (${records.length} records)`);56    return;57  }58  throw new Error('crawl yielded nothing');59}60