spb/rareindex

1/**2 * Live smoke + fixture capture for the auction-house connectors.3 * Usage: pnpm tsx connectors/api/_auction-lib/smoke.ts <connectorId> [--limit N] [--seed URL]... [--category slug]... [--capture name]4 * Requires FIRECRAWL_API_KEY / SCRAPFLY_API_KEY in the environment for the engines the connector uses.5 */6import { createCrawlContext, createRouter, getConnectorMeta, loadConnector } from '@rareindex/connectors';7import { saveFixture } from '@rareindex/connectors/testing';8import { childLogger } from '@rareindex/shared';910const [id, ...rest] = process.argv.slice(2);11if (!id) throw new Error('usage: smoke.ts <connectorId> [--limit N] [--seed URL] [--category slug] [--capture name]');12const opt = { limit: 12, seeds: [] as string[], categories: [] as string[], capture: null as string | null };13for (let i = 0; i < rest.length; i++) {14  const a = rest[i]!;15  if (a === '--limit') opt.limit = Number(rest[++i]);16  else if (a === '--seed') opt.seeds.push(rest[++i]!);17  else if (a === '--category') opt.categories.push(rest[++i]!);18  else if (a === '--capture') opt.capture = rest[++i]!;19}2021const meta = getConnectorMeta(id);22const connector = await loadConnector(id);23const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });24const ctx = createCrawlContext({25  router,26  meta,27  options: { mode: 'probe', limit: opt.limit, ...(opt.seeds.length ? { seeds: opt.seeds } : {}), ...(opt.categories.length ? { categories: opt.categories } : {}) },28  log: childLogger({ connector: id, smoke: true }),29});3031let rawCount = 0;32let total = 0;33const kinds: Record<string, number> = {};34let captured = 0;35for await (const raw of connector.crawl(ctx)) {36  rawCount++;37  const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };38  const records = await connector.normalize(rawLike);39  total += records.length;40  for (const r of records) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;41  const p = raw.payload as { lots?: unknown[]; kind?: string };42  console.log(`raw#${rawCount} ${raw.kind} ${raw.url} → payload.${p.kind} lots=${p.lots?.length ?? '-'} → ${records.length} records`);43  for (const r of records.slice(0, 2)) console.log(JSON.stringify(r, null, 1).slice(0, 1800));44  if (opt.capture && captured < 3 && records.length) {45    const name = `${opt.capture}-${captured + 1}`;46    const first = records[0]!;47    saveFixture(id, name, {48      raw: { ...rawLike, payload: JSON.parse(JSON.stringify(rawLike.payload)) },49      expect: { minCount: 1, kinds: [...new Set(records.map((r) => r.kind))], first: { kind: first.kind, ...('auctionHouse' in first ? { auctionHouse: (first as { auctionHouse: string }).auctionHouse } : {}), ...('currency' in first && first.currency ? { currency: first.currency } : {}) } },50      note: `Captured live by connectors/api/_auction-lib/smoke.ts on ${new Date().toISOString().slice(0, 10)} (${records.length} records from this raw page).`,51    });52    captured++;53    console.log(`  saved fixture data/fixtures/${id}/${name}.json`);54  }55  if (rawCount >= 6) break;56}57console.log(JSON.stringify({ rawCount, totalNormalized: total, kinds, engineStats: ctx.engineStats, anomalies: ctx.anomalies }, null, 1));58