spb/rareindex

1/**2 * Live smoke probe for the g4-shops-intl storefront connectors (no Postgres needed):3 * real router + crawl context, probe mode, limit N (default 5), one shop after another.4 *5 *   pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts totalcards ninoma [--limit 5]6 */7import { readFileSync } from 'node:fs';8import path from 'node:path';9import { ConnectorMetaSchema, createCrawlContext, createRouter, type RareIndexConnector } from '@rareindex/connectors';10import { childLogger } from '@rareindex/shared';1112const args = process.argv.slice(2);13const limitIdx = args.indexOf('--limit');14const limit = limitIdx >= 0 ? Number(args[limitIdx + 1]) : 5;15const ids = args.filter((a, i) => !a.startsWith('--') && (limitIdx < 0 || i !== limitIdx + 1));16const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..');17const router = createRouter({});1819for (const id of ids) {20  const started = Date.now();21  const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(root, 'connectors/api', id, 'meta.json'), 'utf8')));22  const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => RareIndexConnector };23  const connector = mod.default(meta);24  const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) });25  let raws = 0;26  let normalized = 0;27  let skipped = 0;28  const cats = new Map<string, number>();29  let sample = '';30  try {31    for await (const raw of connector.crawl(ctx)) {32      raws++;33      const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });34      if (!out.length) skipped++;35      normalized += out.length;36      for (const r of out) {37        if (r.kind !== 'listing') continue;38        cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1);39        if (!sample) sample = `"${r.rawTitle}" → ${r.attributes.categorySlug} ${r.price} ${r.currency} ${r.availability}${r.grade.grader ? ` ${r.grade.grader} ${r.grade.grade}` : ''}`;40      }41    }42  } catch (e) {43    console.log(`[${id}] CRASH ${(e as Error).message}`);44  }45  console.log(`[${id}] raw=${raws} listings=${normalized} unmapped=${skipped} cats=${JSON.stringify(Object.fromEntries(cats))} anomalies=${JSON.stringify(ctx.anomalies)} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`);46  if (sample) console.log(`  e.g. ${sample}`);47}48