spb/rareindex

1import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';2import { saveFixture } from '@rareindex/connectors/testing';3import createConnector from './index.js';45const save = process.argv.includes('--save');6const meta = getConnectorMeta('novelship');7const connector = createConnector(meta);8const ctx = createCrawlContext({ router: createRouter({}), meta, options: { mode: 'probe', limit: 2, seeds: ['jordan'] } });9let i = 0;10for await (const raw of connector.crawl(ctx)) {11  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });12  const n = (raw.payload as { products: unknown[] }).products.length;13  console.log(`raw ${raw.externalId} products=${n} → ${out.length} records`);14  for (const r of out.slice(0, 3)) console.log('  ', JSON.stringify(r).slice(0, 360));15  if (save && i === 0) saveFixture('novelship', 'jordan-browse-page1', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 30, kinds: ['catalog_item', 'price_observation', 'listing'], requiredFields: ['attributes.identifiers.style_code'] }, note: 'Captured live from novelship.com/sneakers/jordan' });16  i++;17}18const lk = await connector.lookup('https://novelship.com/air-jordan-1-low-midnight-navy-university-blue-553558-404', ctx);19if (lk[0]) {20  const out = await connector.normalize({ ...lk[0], externalId: lk[0].externalId ?? null, fetchedAt: lk[0].fetchedAt ?? new Date() });21  console.log('lookup →', out.length, JSON.stringify(out).slice(0, 500));22  if (save) saveFixture('novelship', 'aj1-low-midnight-navy-product', { raw: { url: lk[0].url, externalId: lk[0].externalId ?? null, kind: lk[0].kind, engine: lk[0].engine, fetchedAt: lk[0].fetchedAt ?? new Date(), payload: lk[0].payload }, expect: { minCount: 1, first: { 'attributes.identifiers.style_code': '553558-404', 'attributes.categorySlug': 'nike_jordan' } }, note: 'Captured live from a novelship.com product page' });23}24console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);25