spb/rareindex

1/**2 * Live capture + smoke for the wave-4 connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc, alt-xyz).3 * Usage: pnpm tsx connectors/api/_lib/capture-s.ts [ids…] [--no-save]4 */5import { createCrawlContext, createRouter, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';6import { localMeta } from './local-meta.js';7import { saveFixture } from '@rareindex/connectors/testing';8import { childLogger } from '@rareindex/shared';910interface Plan {11  dir: string;12  limit: number;13  seeds?: string[];14  pick: (raw: RawRecordInput, i: number) => string | null;15  kinds: string[];16  requiredFields: string[];17}1819const picked = new Set<string>();20const PLANS: Record<string, Plan> = {21  mtgjson: {22    dir: 'api',23    limit: 320,24    seeds: ['LEA'],25    pick: (raw, i) => {26      const p = raw.payload as { card: { name: string }; prices: unknown };27      if (p.card.name === 'Black Lotus') return 'lea-black-lotus';28      if (i === 0) return 'lea-first';29      return null;30    },31    kinds: ['catalog_item', 'price_observation'],32    requiredFields: ['attributes.identifiers.scryfall_id', 'attributes.setCode'],33  },34  cardkingdom: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'single-first' : i === 3 ? 'single-fourth' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.identifiers.cardkingdom_id', 'attributes.set'] },35  myslabs: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'slab-first' : i === 2 ? 'slab-third' : i === 5 ? 'slab-sixth' : null), kinds: ['listing'], requiredFields: ['attributes.identifiers.myslabs_id', 'price'] },36  'tag-pop': { dir: 'firecrawl', limit: 1, seeds: ['https://my.taggrading.com/pop-report/Pokemon/1999/WOTC%20Pok%C3%A9mon?setName=Base%20Set'], pick: () => 'pokemon-1999-base-set', kinds: ['population_report'], requiredFields: ['attributes.set', 'attributes.number'] },37  comc: { dir: 'firecrawl', limit: 1, seeds: ['Cards/Basketball/1986/Fleer'], pick: () => 'basketball-1986-fleer-p1', kinds: ['listing'], requiredFields: ['attributes.set', 'price'] },38  'alt-xyz': {39    dir: 'firecrawl',40    limit: 8,41    pick: (raw) => {42      const p = raw.payload as { transactions: unknown[]; altValue: number | null; listPrice: number | null };43      if (p.transactions.length && !picked.has('item-with-transactions')) return 'item-with-transactions';44      if ((p.altValue || p.listPrice) && !picked.has('item-priced')) return 'item-priced';45      return null;46    },47    kinds: ['sale', 'price_observation', 'listing'],48    requiredFields: ['attributes.identifiers.alt_item_id'],49  },50};5152const args = process.argv.slice(2);53const noSave = args.includes('--no-save');54const ids = args.filter((a) => !a.startsWith('--'));55const targets = ids.length ? ids : Object.keys(PLANS);56const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });57for (const id of targets) {58  const plan = PLANS[id];59  if (!plan) {60    console.log(`[${id}] no plan`);61    continue;62  }63  const started = Date.now();64  const mod = (await import(`../../${plan.dir}/${id}/index.ts`)) as { default: (m: ReturnType<typeof localMeta>) => RareIndexConnector };65  const metaJson = (await import(`../../${plan.dir}/${id}/meta.json`, { with: { type: 'json' } })) as { default: unknown };66  const connector = mod.default(localMeta(metaJson.default));67  const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, ...(plan.seeds ? { seeds: plan.seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });68  let raws = 0;69  let normalized = 0;70  let printed = 0;71  let saved = 0;72  const kinds: Record<string, number> = {};73  try {74    for await (const raw of connector.crawl(ctx)) {75      const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };76      const out = await connector.normalize(rawLike);77      normalized += out.length;78      for (const r of out) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;79      const name = plan.pick(raw, raws);80      raws++;81      if (name && !noSave) {82        picked.add(name);83        saveFixture(id, name, { raw: rawLike, expect: { minCount: 1, kinds: plan.kinds, requiredFields: plan.requiredFields }, note: `Live capture ${new Date().toISOString().slice(0, 10)} from ${raw.url}` });84        saved++;85      }86      if (printed < 3 && out.length) {87        printed++;88        const r = out.find((x) => x.kind === 'sale') ?? out.find((x) => x.kind === 'price_observation') ?? out.find((x) => x.kind === 'population_report') ?? out[0]!;89        const s = r as unknown as Record<string, unknown>;90        const attrs = (s.attributes ?? {}) as Record<string, unknown>;91        console.log(`[${id}] ${r.kind} · ${String(s.rawTitle ?? attrs.name ?? '')} · ${String(attrs.categorySlug)} · set=${String(attrs.set)} #${String(attrs.number)} · grade=${JSON.stringify(s.grade ?? null)} · price=${String(s.price ?? s.total ?? '')} ${String(s.currency ?? '')} · date=${String(s.saleDate ?? s.observationDate ?? s.reportDate ?? '')}`);92      }93    }94  } catch (err) {95    console.log(`[${id}] crawl error: ${err instanceof Error ? err.message : String(err)}`);96  }97  console.log(`[${id}] raw=${raws} normalized=${normalized} kinds=${JSON.stringify(kinds)} saved=${saved} anomalies=${ctx.anomalies.length} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`);98  if (ctx.anomalies.length) console.log(`[${id}] anomalies: ${ctx.anomalies.slice(0, 5).join(' ; ')}`);99}100