/** * Live smoke for the g3-shops-na Shopify store connectors — real router + crawl context, probe mode, * no database needed (pattern of connectors/api/_lib/smoke.ts). * * pnpm tsx connectors/api/_g3-shops-na-lib/probe.ts […] [--limit 3] */ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors'; import { childLogger } from '@rareindex/shared'; import { localMeta } from '../_lib/local-meta.js'; const here = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(here, '../../..'); const args = process.argv.slice(2); const limIdx = args.indexOf('--limit'); const limit = limIdx >= 0 ? Number(args[limIdx + 1]) : 3; const ids = args.filter((a, i) => !a.startsWith('--') && (limIdx < 0 || i !== limIdx + 1)); const router = createRouter({}); for (const id of ids) { const started = Date.now(); const dir = path.join(root, 'connectors/api', id); const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector }; const connector = mod.default(meta); const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit }, log: childLogger({ connector: id, level: 'warn' }) }); let raws = 0; let normalized = 0; const cats = new Map(); let printed = 0; try { for await (const raw of connector.crawl(ctx)) { raws++; const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() }); normalized += out.length; for (const r of out) { if (r.kind !== 'listing') continue; cats.set(r.attributes.categorySlug, (cats.get(r.attributes.categorySlug) ?? 0) + 1); if (printed < 2) { printed++; console.log(` [${id}] ${r.rawTitle.slice(0, 80)} → ${r.price} ${r.currency} ${r.availability} cat=${r.attributes.categorySlug} grade=${r.grade.grader ?? '-'}${r.grade.grade ?? ''} sku=${r.attributes.identifiers.sku ?? '-'}`); } } } } catch (err) { console.error(` [${id}] crawl error: ${(err as Error).message}`); } console.log(`[${id}] raw=${raws} normalized=${normalized} cats=${JSON.stringify(Object.fromEntries(cats))} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)} ms=${Date.now() - started}`); }