spb/rareindex

1/**2 * Build fixtures for the cert connectors from LIVE public verification pages (through the real router,3 * i.e. Firecrawl → Scrapfly exactly as production does). Every fixture is a real capture, trimmed by4 * the connector's own `trim`.5 *6 *   pnpm tsx connectors/firecrawl/_g2-grading-lib/capture.ts <connectorId> <fixtureName>=<certOrUrl> [...]7 *   pnpm tsx connectors/firecrawl/_g2-grading-lib/capture.ts psa-cert charizard-psa10=69211238 not-found=999999998 *9 * Also usable as a live smoke: prints the normalised records of each capture.10 */11import { existsSync, readFileSync, readdirSync } from 'node:fs';12import path from 'node:path';13import { createCrawlContext, createRouter, ConnectorMetaSchema, DomainsFileSchema, DOMAINS_DIR, DOMAINS_PATH, setDomains } from '@rareindex/connectors';14import { saveFixture } from '@rareindex/connectors/testing';15import { childLogger } from '@rareindex/shared';16import type { CertLookupConnector } from './cert-base.js';1718const [connectorId, ...pairs] = process.argv.slice(2);19if (!connectorId || !pairs.length) {20  console.error('usage: capture.ts <connectorId> <fixtureName>=<certOrUrl> ...');21  process.exit(1);22}23try {24  process.loadEnvFile(path.resolve('.env'));25} catch {26  /* .env optional */27}2829// Dev-time guard: while several groups edit connectors/domains.d in parallel, one invalid fragment must30// not stop our captures — load domains.json + every fragment that validates, warn about the others.31{32  const base = DomainsFileSchema.parse(JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')));33  if (existsSync(DOMAINS_DIR)) {34    for (const f of readdirSync(DOMAINS_DIR).filter((x) => x.endsWith('.json')).sort()) {35      const parsed = DomainsFileSchema.safeParse({ ...JSON.parse(readFileSync(path.join(DOMAINS_DIR, f), 'utf8')), version: base.version });36      if (!parsed.success) {37        console.warn(`  [capture] skipping invalid domains fragment ${f}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);38        continue;39      }40      for (const [host, patch] of Object.entries(parsed.data.domains)) base.domains[host] = { ...(base.domains[host] ?? {}), ...patch };41    }42  }43  setDomains(base);44}4546const dir = path.resolve('connectors/firecrawl', connectorId);47const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));48const mod = (await import(path.join(dir, 'index.ts'))) as { default: (m: typeof meta) => CertLookupConnector };49const connector = mod.default(meta);50const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });51const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: pairs.length }, log: childLogger({ connector: connectorId, level: 'warn' }) });5253for (const pair of pairs) {54  const eq = pair.indexOf('=');55  const name = pair.slice(0, eq);56  const seed = pair.slice(eq + 1);57  const cert = /^https?:/.test(seed) ? connector.certFromUrl(seed) : seed;58  if (!cert) {59    console.error(`  [${connectorId}] ${name}: cannot derive a cert from ${seed}`);60    continue;61  }62  const raw = await connector.fetchCert(ctx, cert, { includeNotFound: true });63  if (!raw) {64    console.error(`  [${connectorId}] ${name}: no page captured (${JSON.stringify(ctx.anomalies.slice(-1))})`);65    continue;66  }67  const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });68  const payload = raw.payload as { status: string; url: string };69  const kinds = [...new Set(out.map((r) => r.kind))];70  saveFixture(connectorId, name, {71    raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload },72    // requiredFields must hold for every record kind (population_report has no `grade` block).73    expect: payload.status === 'not_found' ? { count: 0 } : { count: out.length, kinds, requiredFields: ['attributes.categorySlug', `attributes.identifiers.${connector.idKey}`], first: { 'grade.certificationNumber': connector.certIdentifier(cert) } },74    note: payload.status === 'not_found' ? `Live capture (${raw.engine}) of ${payload.url}: the grader reports this certification number as not found; normalises to zero records.` : `Live capture (${raw.engine}) of the public verification page ${payload.url}, trimmed to the results fragment.`,75  });76  console.log(`  [${connectorId}] ${name}: ${payload.status} via ${raw.engine} → ${out.length} record(s) ${kinds.join(',')}`);77  for (const r of out) {78    if (r.kind === 'catalog_item') console.log(`     ${r.rawTitle} | ${r.attributes.categorySlug} | grade=${r.grade.grader} ${r.grade.grade ?? '-'}${r.grade.qualifier ? ` (${r.grade.qualifier})` : ''} | ids=${JSON.stringify(r.attributes.identifiers)}`);79    if (r.kind === 'population_report') console.log(`     population total=${r.total} byGrade=${JSON.stringify(r.byGrade)}`);80  }81}82console.log(`[${connectorId}] engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${ctx.anomalies.length}`);83