spb/rareindex

1import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';2import metaJson from './meta.json' with { type: 'json' };3import { saveFixture } from '@rareindex/connectors/testing';4import createConnector from './index.js';56const save = process.argv.includes('--save');7const meta = ConnectorMetaSchema.parse(metaJson);8const connector = createConnector(meta);9const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });10const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 1 } });11let i = 0;12for await (const raw of connector.crawl(ctx)) {13  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });14  console.log(`raw ${raw.externalId} → ${out.length} records`);15  for (const r of out.slice(0, 4)) console.log('  ', JSON.stringify(r).slice(0, 320));16  if (save && i === 0) {17    const p = raw.payload as any;18    p.products = p.products.slice(0, 8);19    saveFixture('lego-shop', 'star-wars-theme-p1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: p }, expect: { minCount: 8, kinds: ['catalog_item', 'listing'], requiredFields: ['attributes.identifiers.lego_set_number'] }, note: 'Captured live from lego.com/en-us/themes/star-wars, trimmed to 8 products' });20  }21  i++;22}23console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);24