spb/rareindex

1/**2 * Live smoke for the g3-shops-na Shopify store connectors — real router + crawl context, probe mode,3 * no database needed (pattern of connectors/api/_lib/smoke.ts).4 *5 *   pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts <connector-id> [<connector-id>…] [--limit 3]6 */7import { readFileSync } from 'node:fs';8import path from 'node:path';9import { fileURLToPath } from 'node:url';10import { createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors';11import { childLogger } from '@rareindex/shared';12import { localMeta } from '../_lib/local-meta.js';1314const here = path.dirname(fileURLToPath(import.meta.url));15const root = path.resolve(here, '../../..');16const args = process.argv.slice(2);17const limIdx = args.indexOf('--limit');18const limit = limIdx >= 0 ? Number(args[limIdx + 1]) : 3;19const ids = args.filter((a, i) => !a.startsWith('--') && (limIdx < 0 || i !== limIdx + 1));20const router = createRouter({});2122for (const id of ids) {23  const started = Date.now();24  const dir = path.join(root, 'connectors/api', id);25  const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));26  const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector };27  const connector = mod.default(meta);28  const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) });29  let raws = 0;30  let normalized = 0;31  const cats = new Map<string, number>();32  let printed = 0;33  try {34    for await (const raw of connector.crawl(ctx)) {35      raws++;36      const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });37      normalized += out.length;38      for (const r of out) {39        if (r.kind !== 'listing') continue;40        cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1);41        if (printed < 2) {42          printed++;43          console.log(`  [${id}] ${r.rawTitle.slice(0, 80)} → ${r.price} ${r.currency} ${r.availability} cat=${r.attributes.categorySlug} grade=${r.grade.grader ?? '-'}${r.grade.grade ?? ''} sku=${r.attributes.identifiers.sku ?? '-'}`);44        }45      }46    }47  } catch (err) {48    console.error(`  [${id}] crawl error: ${(err as Error).message}`);49  }50  console.log(`[${id}] raw=${raws} normalized=${normalized} cats=${JSON.stringify(Object.fromEntries(cats))} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)} ms=${Date.now() - started}`);51}52