spb/rareindex

1/**2 * Live smoke for the g1-cards-eu-jp connectors: real router + crawl context, probe mode, small limits,3 * seeds chosen to keep traffic (and Scrapfly credits) minimal. Postgres is not needed.4 * Usage: pnpm tsx connectors/api/_g1-cards-eu-jp-lib/smoke.ts [ids…]5 */6import { createCrawlContext, createRouter, loadConnector } from '@rareindex/connectors';7import { childLogger } from '@rareindex/shared';89try {10  process.loadEnvFile('.env');11} catch {12  /* no .env */13}1415const PLAN: Record<string, { limit: number; seeds?: string[] }> = {16  'cardmarket-priceguide': { limit: 5, seeds: ['18'] }, // One Piece: smallest files (~4 MB)17  'limitless-tcg': { limit: 1, seeds: ['MEG', 'M6'] },18  'yuyu-tei': { limit: 1, seeds: ['poc'] },19  hareruya: { limit: 1 },20  magi: { limit: 1, seeds: ['31'] },21  cardrush: { limit: 1, seeds: ['www.cardrush-mtg.jp'] }, // plain-HTTP store only (no Scrapfly credits)22  cardtrader: { limit: 1 },23};2425const ids = process.argv.slice(2).length ? process.argv.slice(2) : Object.keys(PLAN);26const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });27for (const id of ids) {28  const started = Date.now();29  const connector = await loadConnector(id);30  const plan = PLAN[id] ?? { limit: 1 };31  const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, seeds: plan.seeds }, log: childLogger({ connector: id, level: 'warn' }) });32  let raws = 0;33  let normalized = 0;34  const kinds: Record<string, number> = {};35  let printed = 0;36  try {37    for await (const raw of connector.crawl(ctx)) {38      raws++;39      const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });40      normalized += out.length;41      for (const r of out) {42        kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;43        if (printed < 3) {44          printed++;45          const summary =46            r.kind === 'price_observation'47              ? `${r.priceKind} ${r.price} ${r.currency} @ ${r.observationDate.toISOString().slice(0, 10)}`48              : r.kind === 'listing'49                ? `${r.price} ${r.currency} ${r.availability} ids=${JSON.stringify(r.attributes.identifiers)}`50                : r.kind === 'catalog_item'51                  ? `set=${r.attributes.set ?? '-'} #${r.attributes.number ?? '-'} variant=${r.attributes.variant ?? '-'} ids=${JSON.stringify(r.attributes.identifiers)}`52                  : '';53          console.log(`  [${id}] ${r.kind}: ${'rawTitle' in r ? r.rawTitle : ''} → ${summary}`);54        }55      }56    }57  } catch (err) {58    console.log(`  [${id}] crawl error: ${err instanceof Error ? err.message : String(err)}`);59  }60  console.log(`[${id}] raw=${raws} normalized=${normalized} kinds=${JSON.stringify(kinds)} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)} ms=${Date.now() - started}`);61}62