spb/rareindex

1/**2 * Live smoke + fixture capture for the g9 connectors (real router + crawl context, no database).3 *4 *   pnpm tsx connectors/api/_g9-asia-watch-sneaker-lib/capture.ts <connectorId> [--limit 1] [--seeds a,b] [--mode probe|backfill]5 *        [--name fixtureName] [--trim 12] [--no-save] [--cursor '{"json":1}'] [--lookup <url>]6 *7 * Payload arrays (rows/items/tiles/data/list/offers) are trimmed to --trim entries before saving so fixtures stay small8 * but remain genuine live captures.9 */10import { existsSync, readFileSync } from 'node:fs';11import path from 'node:path';12import { ConnectorMetaSchema, DOMAINS_DIR, DOMAINS_PATH, createCrawlContext, createRouter, loadDomains, setDomains, type ConnectorMeta, type RareIndexConnector } from '@rareindex/connectors';13import { saveFixture } from '@rareindex/connectors/testing';1415/** Another group's domains.d fragment may be temporarily invalid; fall back to domains.json + our own fragment so smoke runs stay possible. */16function ensureDomains() {17  try {18    loadDomains();19  } catch (err) {20    console.warn(`[capture] domains.d contains an invalid fragment (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); using domains.json + g9 fragment only`);21    const base = JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')) as { version?: string; defaults?: unknown; domains?: Record<string, unknown> };22    const frag = JSON.parse(readFileSync(path.join(DOMAINS_DIR, 'g9-asia-watch-sneaker.json'), 'utf8')) as { domains?: Record<string, unknown> };23    setDomains({ version: base.version ?? '1.0', defaults: (base.defaults ?? {}) as never, domains: { ...(base.domains ?? {}), ...(frag.domains ?? {}) } as never });24  }25}2627function loadEnv() {28  const p = path.resolve(process.cwd(), '.env');29  if (!existsSync(p)) return;30  for (const line of readFileSync(p, 'utf8').split('\n')) {31    const m = line.match(/^([A-Z0-9_]+)=(.*)$/);32    if (m && !process.env[m[1]!]) process.env[m[1]!] = m[2]!.replace(/^"(.*)"$/, '$1');33  }34}3536function flag(name: string): string | undefined {37  const i = process.argv.indexOf(`--${name}`);38  return i >= 0 ? process.argv[i + 1] : undefined;39}4041function trimPayload(payload: unknown, n: number): unknown {42  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return payload;43  const out: Record<string, unknown> = { ...(payload as Record<string, unknown>) };44  for (const k of ['rows', 'items', 'tiles', 'data', 'list', 'offers', 'cards', 'hits', 'Items', 'products']) {45    if (Array.isArray(out[k]) && (out[k] as unknown[]).length > n) out[k] = (out[k] as unknown[]).slice(0, n);46  }47  return out;48}4950async function main() {51  loadEnv();52  ensureDomains();53  const id = process.argv[2];54  if (!id || id.startsWith('--')) throw new Error('usage: capture.ts <connectorId> [--limit N] [--seeds a,b] [--name x] [--trim N] [--no-save] [--lookup url]');55  const limit = Number(flag('limit') ?? 1);56  const trim = Number(flag('trim') ?? 12);57  const save = !process.argv.includes('--no-save');58  const seeds = flag('seeds')?.split(',').map((s) => s.trim()).filter(Boolean);59  const mode = (flag('mode') ?? 'probe') as 'probe' | 'incremental' | 'backfill';60  const cursor = flag('cursor') ? (JSON.parse(flag('cursor')!) as Record<string, unknown>) : undefined;61  const dir = ['api', 'firecrawl', 'scrapfly'].map((d) => path.resolve('connectors', d, id)).find((d) => existsSync(path.join(d, 'meta.json')));62  if (!dir) throw new Error(`no meta.json for ${id}`);63  const metaRaw = JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')) as Record<string, unknown>;64  const meta: ConnectorMeta = ConnectorMetaSchema.parse({ ...metaRaw, module: metaRaw.module ?? `${path.basename(path.dirname(dir))}/${id}` });65  const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: ConnectorMeta) => RareIndexConnector };66  const connector = mod.default(meta);67  const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });68  const ctx = createCrawlContext({ router, meta, options: { mode, limit, seeds, cursor } });69  const lookupUrl = flag('lookup');70  const raws = lookupUrl ? await connector.lookup!(lookupUrl, ctx) : null;71  let n = 0;72  const iter = raws ? (async function* () { for (const r of raws) yield r; })() : connector.crawl(ctx);73  for await (const raw of iter) {74    n++;75    const records = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });76    const kinds = records.reduce<Record<string, number>>((acc, r) => ((acc[r.kind] = (acc[r.kind] ?? 0) + 1), acc), {});77    console.log(`\n[${id}] raw #${n} ${raw.url} engine=${raw.engine} http=${raw.httpStatus ?? '-'} → ${records.length} records ${JSON.stringify(kinds)}`);78    for (const r of records.slice(0, 4)) {79      const a = r as Record<string, unknown>;80      const price = 'price' in r ? `${(r as { price: unknown }).price} ${(r as { currency?: string }).currency ?? ''}` : 'currentBid' in r ? `bid ${(r as { currentBid: unknown }).currentBid} ${(r as { currency?: string }).currency ?? ''}` : '';81      const date = 'saleDate' in r ? (r as { saleDate: Date }).saleDate.toISOString().slice(0, 10) : 'endsAt' in r && (r as { endsAt: Date | null }).endsAt ? `ends ${(r as { endsAt: Date }).endsAt.toISOString().slice(0, 10)}` : '';82      const attrs = 'attributes' in r ? JSON.stringify({ cat: r.attributes.categorySlug, brand: r.attributes.brand, ref: r.attributes.reference, ids: r.attributes.identifiers }).slice(0, 200) : '';83      console.log('  -', r.kind, '|', String((a.rawTitle as string | undefined) ?? (a.title as string | undefined) ?? '').slice(0, 80), '|', price, '|', date, '|', attrs, '|', 'grade' in r && r.grade.grade ? `${r.grade.grader} ${r.grade.grade}` : '');84    }85    if (save) {86      const name = flag('name') ? (n === 1 ? flag('name')! : `${flag('name')}-${n}`) : (raw.externalId ?? `raw-${n}`).replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').toLowerCase().slice(0, 80);87      const payload = trimPayload(raw.payload, trim);88      const trimmedRecords = await connector.normalize({ ...raw, payload, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });89      saveFixture(id, name, {90        raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload },91        expect: { minCount: Math.min(trimmedRecords.length, 1), kinds: [...new Set(trimmedRecords.map((r) => r.kind))] },92        note: `Captured live from ${raw.url} by connectors/api/_g9-asia-watch-sneaker-lib/capture.ts (payload arrays trimmed to ${trim} entries).`,93      });94      console.log(`  saved fixture data/fixtures/${id}/${name}.json (${trimmedRecords.length} records after trim)`);95    }96  }97  console.log(`\n[${id}] done: ${n} raw records; engine stats ${JSON.stringify(ctx.engineStats)}; anomalies ${JSON.stringify(ctx.anomalies)}`);98}99100main().catch((e) => {101  console.error(e);102  process.exit(1);103});104