/** * Live smoke probe for the g4-shops-intl storefront connectors (no Postgres needed): * real router + crawl context, probe mode, limit N (default 5), one shop after another. * * pnpm tsx connectors/api/_g4-shops-intl-lib/smoke.ts totalcards ninoma [--limit 5] */ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { ConnectorMetaSchema, createCrawlContext, createRouter, type RareIndexConnector } from '@rareindex/connectors'; import { childLogger } from '@rareindex/shared'; const args = process.argv.slice(2); const limitIdx = args.indexOf('--limit'); const limit = limitIdx >= 0 ? Number(args[limitIdx + 1]) : 5; const ids = args.filter((a, i) => !a.startsWith('--') && (limitIdx < 0 || i !== limitIdx + 1)); const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); const router = createRouter({}); for (const id of ids) { const started = Date.now(); const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(root, 'connectors/api', id, 'meta.json'), 'utf8'))); const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => RareIndexConnector }; const connector = mod.default(meta); const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) }); let raws = 0; let normalized = 0; let skipped = 0; const cats = new Map(); let sample = ''; try { for await (const raw of connector.crawl(ctx)) { raws++; const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); if (!out.length) skipped++; normalized += out.length; for (const r of out) { if (r.kind !== 'listing') continue; cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1); if (!sample) sample = `"${r.rawTitle}" → ${r.attributes.categorySlug} ${r.price} ${r.currency} ${r.availability}${r.grade.grader ? ` ${r.grade.grader} ${r.grade.grade}` : ''}`; } } } catch (e) { console.log(`[${id}] CRASH ${(e as Error).message}`); } console.log(`[${id}] raw=${raws} listings=${normalized} unmapped=${skipped} cats=${JSON.stringify(Object.fromEntries(cats))} anomalies=${JSON.stringify(ctx.anomalies)} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`); if (sample) console.log(` e.g. ${sample}`); }