spb/rareindex

1/**2 * Live smoke + fixture capture for the g7 North-American auction connectors (works before the registry is3 * rebuilt: the connector's own meta.json is used).4 * Usage: pnpm tsx connectors/api/_g7-auctions-na-lib/smoke.ts <engine>/<id> [--limit N] [--seed URL]... [--capture name] [--trim N] [--mode probe|incremental|backfill]5 */6import { readFileSync } from 'node:fs';7import path from 'node:path';8import { pathToFileURL } from 'node:url';9import { ConnectorMetaSchema, createCrawlContext, createRouter, DomainsFileSchema, setDomains, type ConnectorFactory, type ConnectorMeta } from '@rareindex/connectors';10import { saveFixture } from '@rareindex/connectors/testing';11import { childLogger } from '@rareindex/shared';1213// Parallel connector agents may leave an invalid fragment in connectors/domains.d/ which would make14// loadDomains() throw for everybody; pre-seed the policy cache from domains.json + this group's fragment.15{16  const base = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.json'), 'utf8')) as { version?: string; defaults?: unknown; domains?: Record<string, unknown> };17  const frag = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.d/g7-auctions-na.json'), 'utf8')) as { domains?: Record<string, unknown> };18  setDomains(DomainsFileSchema.parse({ ...base, domains: { ...(base.domains ?? {}), ...(frag.domains ?? {}) } }));19}2021const [modulePath, ...rest] = process.argv.slice(2);22if (!modulePath) throw new Error('usage: smoke.ts <engine>/<id> [--limit N] [--seed URL] [--capture name] [--trim N] [--mode m]');23const opt = { limit: 3, seeds: [] as string[], capture: null as string | null, trim: 12, mode: 'probe' as 'probe' | 'incremental' | 'backfill', maxRaw: 4 };24for (let i = 0; i < rest.length; i++) {25  const a = rest[i]!;26  if (a === '--limit') opt.limit = Number(rest[++i]);27  else if (a === '--seed') opt.seeds.push(rest[++i]!);28  else if (a === '--capture') opt.capture = rest[++i]!;29  else if (a === '--trim') opt.trim = Number(rest[++i]);30  else if (a === '--mode') opt.mode = rest[++i] as typeof opt.mode;31  else if (a === '--max-raw') opt.maxRaw = Number(rest[++i]);32}3334const dir = path.resolve(process.cwd(), 'connectors', modulePath);35const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));36const factory = (await import(pathToFileURL(path.join(dir, 'index.ts')).href)).default as (meta: ConnectorMeta) => ReturnType<ConnectorFactory>;37const connector = await factory(meta);38const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });39const ctx = createCrawlContext({ router, meta, options: { mode: opt.mode, limit: opt.limit, ...(opt.seeds.length ? { seeds: opt.seeds } : {}) }, log: childLogger({ connector: meta.id, smoke: true }), onCursor: async (c) => console.log('cursor →', JSON.stringify(c).slice(0, 300)) });4041/** Trim list-shaped payloads so fixtures stay small (keeps every scalar field, first N items). */42function trim(payload: unknown): unknown {43  if (!payload || typeof payload !== 'object') return payload;44  const p = { ...(payload as Record<string, unknown>) };45  for (const k of Object.keys(p)) if (Array.isArray(p[k]) && (p[k] as unknown[]).length > opt.trim) p[k] = (p[k] as unknown[]).slice(0, opt.trim);46  return p;47}4849let rawCount = 0;50let total = 0;51let captured = 0;52const kinds: Record<string, number> = {};53for await (const raw of connector.crawl(ctx)) {54  rawCount++;55  const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };56  const records = await connector.normalize(rawLike);57  total += records.length;58  for (const r of records) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;59  console.log(`raw#${rawCount} ${raw.kind} ${raw.url} → ${records.length} records`);60  for (const r of records.slice(0, 2)) console.log(JSON.stringify(r).slice(0, 900));61  if (opt.capture && captured < 3 && records.length) {62    const trimmed = trim(JSON.parse(JSON.stringify(rawLike.payload)));63    const recs = await connector.normalize({ ...rawLike, payload: trimmed });64    const first = recs[0]!;65    const name = `${opt.capture}-${captured + 1}`;66    saveFixture(meta.id, name, {67      raw: { ...rawLike, payload: trimmed },68      expect: { minCount: 1, kinds: [...new Set(recs.map((r) => r.kind))], first: { kind: first.kind, ...('auctionHouse' in first && first.auctionHouse ? { auctionHouse: first.auctionHouse } : {}), ...('currency' in first && first.currency ? { currency: first.currency } : {}) } },69      note: `Captured live by connectors/api/_g7-auctions-na-lib/smoke.ts on ${new Date().toISOString().slice(0, 10)} from ${raw.url} (payload lists trimmed to ${opt.trim} items; ${recs.length} records).`,70    });71    captured++;72    console.log(`  saved fixture data/fixtures/${meta.id}/${name}.json`);73  }74  if (rawCount >= opt.maxRaw) break;75}76console.log(JSON.stringify({ rawCount, totalNormalized: total, kinds, engineStats: ctx.engineStats, anomalies: ctx.anomalies }, null, 1));77