spb/rareindex

1/**2 * Live capture + smoke for the luxury-wave connectors. Runs a probe crawl through the real router,3 * prints normalised samples and saves the first raw record(s) as fixtures.4 *5 *   pnpm tsx connectors/api/_luxury-lib/capture.ts <connectorId> [limit=1] [--no-save]6 */7import { readFileSync, existsSync } from 'node:fs';8import path from 'node:path';9import { createCrawlContext, createRouter, ConnectorMetaSchema } from '@rareindex/connectors';10import { saveFixture } from '@rareindex/connectors/testing';11import type { RareIndexConnector, ConnectorMeta } from '@rareindex/connectors';1213function loadEnv() {14  const p = path.resolve(process.cwd(), '.env');15  if (!existsSync(p)) return;16  for (const line of readFileSync(p, 'utf8').split('\n')) {17    const m = line.match(/^([A-Z0-9_]+)=(.*)$/);18    if (m && !process.env[m[1]!]) process.env[m[1]!] = m[2]!.replace(/^"(.*)"$/, '$1');19  }20}2122async function main() {23  loadEnv();24  const id = process.argv[2];25  if (!id) throw new Error('usage: capture.ts <connectorId> [limit] [--no-save]');26  const limit = Number(process.argv[3] ?? 1);27  const save = !process.argv.includes('--no-save');28  const dirs = ['api', 'firecrawl', 'scrapfly'];29  const dir = dirs.map((d) => path.resolve('connectors', d, id)).find((d) => existsSync(path.join(d, 'meta.json')));30  if (!dir) throw new Error(`no meta.json for ${id}`);31  const metaRaw = JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'));32  const meta: ConnectorMeta = ConnectorMetaSchema.parse({ ...metaRaw, module: metaRaw.module ?? `${path.basename(path.dirname(dir))}/${id}` });33  const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector };34  const connector = mod.default(meta);35  const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });36  const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit } });37  let n = 0;38  for await (const raw of connector.crawl(ctx)) {39    n++;40    const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });41    const kinds = records.reduce<Record<string, number>>((acc, r) => ((acc[r.kind] = (acc[r.kind] ?? 0) + 1), acc), {});42    console.log(`\n[${id}] raw #${n} ${raw.url} engine=${raw.engine} → ${records.length} records ${JSON.stringify(kinds)}`);43    for (const r of records.slice(0, 3)) {44      const anyR = r as Record<string, unknown>;45      console.log('  -', r.kind, '|', (anyR.rawTitle as string | undefined)?.slice(0, 90) ?? (anyR.title as string | undefined), '|', 'price' in r ? `${(r as { price: unknown }).price} ${(r as { currency?: string }).currency ?? ''}` : '', '|', 'saleDate' in r ? (r as { saleDate: Date }).saleDate.toISOString().slice(0, 10) : '', '|', 'attributes' in r ? JSON.stringify({ cat: r.attributes.categorySlug, brand: r.attributes.brand, model: r.attributes.model, ref: r.attributes.reference, ids: r.attributes.identifiers }).slice(0, 220) : '');46    }47    if (save) {48      const name = `${(raw.externalId ?? `raw-${n}`).replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').toLowerCase()}`;49      saveFixture(id, name, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: Math.min(records.length, 1), kinds: [...new Set(records.map((r) => r.kind))] }, note: 'Captured live by connectors/api/_luxury-lib/capture.ts' });50      console.log(`  saved fixture ${name}`);51    }52  }53  console.log(`\n[${id}] done: ${n} raw records; engine stats ${JSON.stringify(ctx.engineStats)}; anomalies ${JSON.stringify(ctx.anomalies)}`);54}5556main().catch((e) => {57  console.error(e);58  process.exit(1);59});60