spb/rareindex

1/**2 * Live capture + smoke for the wave-3 TCG connectors. Runs each connector in probe mode through the real3 * router/crawl context, saves small fixtures and prints sample normalized records.4 * Usage: pnpm tsx connectors/api/_lib/capture-k.ts [ids…]   (default: all seven)5 */6import { createCrawlContext, createRouter, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';7import { localMeta } from './local-meta.js';8import { saveFixture } from '@rareindex/connectors/testing';9import { childLogger } from '@rareindex/shared';1011interface Plan {12  limit: number;13  seeds?: string[];14  /** pick which raw records become fixtures: returns a fixture name or null */15  pick: (raw: RawRecordInput, i: number) => string | null;16  kinds: string[];17  requiredFields: string[];18}1920const PLANS: Record<string, Plan> = {21  tcgcsv: {22    limit: 40,23    seeds: ['3'],24    pick: (raw, i) => {25      const p = raw.payload as { product: { name: string }; prices: unknown[] };26      if (i === 0) return 'pokemon-first-product';27      if (p.prices.length > 1 && /charizard|blastoise|venusaur/i.test(p.product.name)) return 'pokemon-multi-subtype';28      return null;29    },30    kinds: ['catalog_item', 'price_observation'],31    requiredFields: ['attributes.identifiers.tcgplayer_id', 'attributes.set'],32  },33  digimoncard: { limit: 12, seeds: ['BT5'], pick: (_r, i) => (i === 0 ? 'bt5-first' : i === 5 ? 'bt5-sixth' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.digimoncard_id', 'attributes.number'] },34  'swu-db': { limit: 8, seeds: ['SOR'], pick: (_r, i) => (i === 0 ? 'sor-first' : i === 3 ? 'sor-fourth' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.setCode', 'attributes.number'] },35  'grand-archive': { limit: 6, pick: (_r, i) => (i === 0 ? 'page1-first' : i === 2 ? 'page1-third' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.gatcg_edition_id'] },36  'sorcery-tcg': { limit: 6, pick: (_r, i) => (i === 0 ? 'first' : i === 5 ? 'sixth' : null), kinds: ['catalog_item'], requiredFields: ['attributes.identifiers.sorcery_printing_id', 'attributes.set'] },37  mtggoldfish: {38    limit: 400,39    seeds: ['Limited+Edition+Alpha'],40    pick: (raw) => {41      const p = raw.payload as { kind: string; card: { display_name: string; foil?: boolean } };42      if (p.kind === 'card' && p.card.display_name === 'Black Lotus' && !p.card.foil) return 'lea-black-lotus';43      if (p.kind === 'history' && p.card.display_name === 'Black Lotus') return 'lea-black-lotus-history';44      if (p.kind === 'card' && p.card.display_name === 'Air Elemental') return 'lea-air-elemental';45      return null;46    },47    kinds: ['catalog_item', 'price_observation'],48    requiredFields: ['attributes.setCode', 'attributes.number'],49  },50  pokemonprice: { limit: 3, seeds: ['base-set-1st-edition'], pick: (raw, i) => (i === 0 ? 'base-set-1st-edition-first' : i === 2 ? 'base-set-1st-edition-third' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.set', 'attributes.number'] },51};5253const ids = process.argv.slice(2).length ? process.argv.slice(2) : Object.keys(PLANS);54const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });55for (const id of ids) {56  const plan = PLANS[id];57  if (!plan) {58    console.log(`[${id}] no plan`);59    continue;60  }61  const started = Date.now();62  const mod = (await import(`../${id}/index.ts`)) as { default: (m: ReturnType<typeof localMeta>) => RareIndexConnector };63  const metaJson = (await import(`../${id}/meta.json`, { with: { type: 'json' } })) as { default: unknown };64  const connector = mod.default(localMeta(metaJson.default));65  const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, ...(plan.seeds ? { seeds: plan.seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });66  let raws = 0;67  let normalized = 0;68  let printed = 0;69  let saved = 0;70  for await (const raw of connector.crawl(ctx)) {71    const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };72    const out = await connector.normalize(rawLike);73    normalized += out.length;74    const name = plan.pick(raw, raws);75    raws++;76    if (name) {77      saveFixture(id, name, { raw: rawLike, expect: { minCount: 1, kinds: plan.kinds, requiredFields: plan.requiredFields }, note: `Live capture ${new Date().toISOString().slice(0, 10)} from ${raw.url}` });78      saved++;79    }80    if (printed < 3) {81      for (const r of out.slice(0, 3)) {82        const summary = r.kind === 'price_observation' ? `${r.priceKind} ${r.price} ${r.currency} @ ${r.observationDate.toISOString().slice(0, 10)}${r.grade.grader ? ` [${r.grade.grader} ${r.grade.grade ?? ''}]` : ''}` : r.kind === 'catalog_item' ? `variant=${r.attributes.variant ?? '-'} set=${r.attributes.set ?? '-'} #${r.attributes.number ?? '-'} ids=${JSON.stringify(r.attributes.identifiers)}` : '';83        console.log(`  [${id}] ${r.kind}: ${'rawTitle' in r ? r.rawTitle : ''} → ${summary}`);84      }85      printed++;86    }87  }88  console.log(`[${id}] raw=${raws} normalized=${normalized} fixtures=${saved} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${ctx.anomalies.slice(0, 3).join(' | ')} ms=${Date.now() - started}`);89}90