/** * Live smoke + fixture capture for the auction-house connectors. * Usage: pnpm tsx connectors/api/_auction-lib/smoke.ts [--limit N] [--seed URL]... [--category slug]... [--capture name] * Requires FIRECRAWL_API_KEY / SCRAPFLY_API_KEY in the environment for the engines the connector uses. */ import { createCrawlContext, createRouter, getConnectorMeta, loadConnector } from '@rareindex/connectors'; import { saveFixture } from '@rareindex/connectors/testing'; import { childLogger } from '@rareindex/shared'; const [id, ...rest] = process.argv.slice(2); if (!id) throw new Error('usage: smoke.ts [--limit N] [--seed URL] [--category slug] [--capture name]'); const opt = { limit: 12, seeds: [] as string[], categories: [] as string[], capture: null as string | null }; for (let i = 0; i < rest.length; i++) { const a = rest[i]!; if (a === '--limit') opt.limit = Number(rest[++i]); else if (a === '--seed') opt.seeds.push(rest[++i]!); else if (a === '--category') opt.categories.push(rest[++i]!); else if (a === '--capture') opt.capture = rest[++i]!; } const meta = getConnectorMeta(id); const connector = await loadConnector(id); const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY }); const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: opt.limit, ...(opt.seeds.length ? { seeds: opt.seeds } : {}), ...(opt.categories.length ? { categories: opt.categories } : {}) }, log: childLogger({ connector: id, smoke: true }), }); let rawCount = 0; let total = 0; const kinds: Record = {}; let captured = 0; for await (const raw of connector.crawl(ctx)) { rawCount++; const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }; const records = await connector.normalize(rawLike); total += records.length; for (const r of records) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1; const p = raw.payload as { lots?: unknown[]; kind?: string }; console.log(`raw#${rawCount} ${raw.kind} ${raw.url} → payload.${p.kind} lots=${p.lots?.length ?? '-'} → ${records.length} records`); for (const r of records.slice(0, 2)) console.log(JSON.stringify(r, null, 1).slice(0, 1800)); if (opt.capture && captured < 3 && records.length) { const name = `${opt.capture}-${captured + 1}`; const first = records[0]!; saveFixture(id, name, { raw: { ...rawLike, payload: JSON.parse(JSON.stringify(rawLike.payload)) }, expect: { minCount: 1, kinds: [...new Set(records.map((r) => r.kind))], first: { kind: first.kind, ...('auctionHouse' in first ? { auctionHouse: (first as { auctionHouse: string }).auctionHouse } : {}), ...('currency' in first && first.currency ? { currency: first.currency } : {}) } }, note: `Captured live by connectors/api/_auction-lib/smoke.ts on ${new Date().toISOString().slice(0, 10)} (${records.length} records from this raw page).`, }); captured++; console.log(` saved fixture data/fixtures/${id}/${name}.json`); } if (rawCount >= 6) break; } console.log(JSON.stringify({ rawCount, totalNormalized: total, kinds, engineStats: ctx.engineStats, anomalies: ctx.anomalies }, null, 1));