spb/rareindex

1import { createRouter, createCrawlContext, getConnectorMeta } from '@rareindex/connectors';2import { saveFixture } from '@rareindex/connectors/testing';3import createConnector from './index.js';45const save = process.argv.includes('--save');6const meta = getConnectorMeta('pcgs-priceguide');7const connector = createConnector(meta);8const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });9const ctx = createCrawlContext({ router, meta, options: { mode: 'backfill', limit: 1, seeds: ['morgan-dollar/744'] } });10for await (const raw of connector.crawl(ctx)) {11  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });12  const p = raw.payload as { rows: unknown[]; grades: string[]; lastUpdate: string | null };13  console.log(`raw ${raw.externalId} (${raw.engine}) rows=${p.rows.length} grades=${p.grades.join(',')} lastUpdate=${p.lastUpdate} → ${out.length} records`);14  for (const r of out.slice(0, 4)) console.log('  ', JSON.stringify(r).slice(0, 330));15  if (save) {16    // keep the fixture small: first 12 rows17    const small = { ...(raw.payload as object), rows: p.rows.slice(0, 12) };18    saveFixture('pcgs-priceguide', 'morgan-dollar-ms', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: small }, expect: { minCount: 20, kinds: ['catalog_item', 'price_observation'], first: { 'attributes.categorySlug': 'coins', 'attributes.series': 'Morgan Dollar' } }, note: 'Captured live via Firecrawl; rows truncated to 12' });19  }20}21console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);22