spb/rareindex

1/**2 * Live smoke + fixture capture for the g8 auction connectors (plain HTTP, no paid engines unless the3 * connector's meta says so). Runs the real router/context in probe mode and saves the first raw records.4 * Usage: pnpm tsx connectors/api/_g8-auctions-eu-apac-lib/capture.ts <connectorId> [--save] [--limit N] [--name prefix]5 */6import { readFileSync } from 'node:fs';7import path from 'node:path';8import { ConnectorMetaSchema, DomainsFileSchema, DOMAINS_PATH, createCrawlContext, createRouter, setDomains, type RareIndexConnector } from '@rareindex/connectors';9import { saveFixture } from '@rareindex/connectors/testing';10import { childLogger } from '@rareindex/shared';1112// Load domains.json + only this group's fragment (other groups' work-in-progress fragments may be invalid while agents run in parallel).13{14  const base = DomainsFileSchema.parse(JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')));15  const frag = JSON.parse(readFileSync(path.resolve(process.cwd(), 'connectors/domains.d/g8-auctions-eu-apac.json'), 'utf8')) as { domains: Record<string, unknown> };16  setDomains({ ...base, domains: { ...base.domains, ...(frag.domains as typeof base.domains) } });17}1819const [id, ...rest] = process.argv.slice(2);20if (!id) throw new Error('usage: capture.ts <connectorId> [--save] [--limit N] [--name prefix]');21const save = rest.includes('--save');22const limit = rest.includes('--limit') ? Number(rest[rest.indexOf('--limit') + 1]) : 2;23const prefix = rest.includes('--name') ? rest[rest.indexOf('--name') + 1]! : 'sale';2425const dir = path.resolve(process.cwd(), 'connectors/api', id);26const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));27const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: typeof meta) => RareIndexConnector };28const connector = mod.default(meta);29const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });30const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) });3132let i = 0;33const started = Date.now();34for await (const raw of connector.crawl(ctx)) {35  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });36  const sales = out.filter((r) => r.kind === 'sale');37  const lots = out.filter((r) => r.kind === 'auction_lot');38  const currencies = [...new Set(sales.map((r) => (r.kind === 'sale' ? r.currency : '')))];39  console.log(`raw ${raw.externalId} (${raw.engine}, ${raw.httpStatus}) → ${out.length} records: ${sales.length} sales ${JSON.stringify(currencies)}, ${lots.length} auction_lot`);40  for (const r of out.slice(0, 2)) console.log('  ', JSON.stringify(r).slice(0, 360));41  if (save) {42    const name = `${prefix}-${String(raw.externalId ?? i).replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`;43    saveFixture(meta.id, name, {44      raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload },45      expect: { minCount: Math.max(1, Math.min(5, out.length)), kinds: ['sale', 'auction_lot'], requiredFields: ['rawTitle', 'attributes.categorySlug'] },46      note: `Live capture via the real router (engine ${raw.engine}) from ${raw.url} on ${new Date().toISOString().slice(0, 10)}; payload trimmed by the connector's own parser.`,47    });48    console.log(`  saved fixture ${name}`);49  }50  i++;51}52console.log(`done raw=${i} ms=${Date.now() - started} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`);53