spb/rareindex

1import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';2import { saveFixture } from '@rareindex/connectors/testing';3import metaJson from './meta.json' with { type: 'json' };4import createConnector from './index.js';56const save = process.argv.includes('--save');7const meta = ConnectorMetaSchema.parse(metaJson);8const connector = createConnector(meta);9const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });10const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2, categories: ['contemporary_art'] } });11let i = 0;12for await (const raw of connector.crawl(ctx)) {13  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });14  console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} listings`);15  for (const r of out.slice(0, 3)) console.log('  ', JSON.stringify(r).slice(0, 420));16  if (save && i < 2) saveFixture(meta.id, i === 0 ? 'paintings-page-1' : 'paintings-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 20, kinds: ['listing'], requiredFields: ['price', 'currency', 'attributes.brand'] }, note: 'Captured live from saatchiart.com __NEXT_DATA__' });17  i++;18}19const looked = await connector.lookup('https://www.saatchiart.com/art/Painting-Octhopuses/626451/10544327/view', ctx);20console.log('lookup →', looked.length ? JSON.stringify((await connector.normalize({ ...looked[0]!, externalId: looked[0]!.externalId ?? null, fetchedAt: new Date() }))[0]).slice(0, 300) : 'none');21console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);22