${(snapshot.match(/<title>([^<]*)/)?.[1] ?? '').trim()}

/** * Live fixture capture for the g6 connectors (real pages/APIs, payloads trimmed, snapshots cut down). * Usage: pnpm tsx connectors/api/_g6-comics-toys-games-lib/capture.ts [limit] [seed...] * e.g. … capture.ts videogametrader 2 * … capture.ts mycomicshop 1 78991 * … capture.ts entertainment-earth 1 */ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { createCrawlContext, createRouter, type RawRecordInput } from '@rareindex/connectors'; import { saveFixture } from '@rareindex/connectors/testing'; import { childLogger } from '@rareindex/shared'; import { localMeta } from '../_lib/local-meta.js'; import { loadEnv, useGroupDomains } from './capture-env.js'; loadEnv(); useGroupDomains(); const here = path.dirname(fileURLToPath(import.meta.url)); const connectorsDir = path.resolve(here, '../..'); const [id, limitArg, ...seeds] = process.argv.slice(2); if (!id) throw new Error('usage: capture.ts [limit] [seed...]'); const MODULES: Record = { mycomicshop: 'scrapfly/mycomicshop', 'entertainment-earth': 'scrapfly/entertainment-earth', 'miniature-market': 'api/miniature-market', videogametrader: 'api/videogametrader', 'mattel-creations': 'api/mattel-creations', comiclink: 'scrapfly/comiclink', gcd: 'api/gcd', estarland: 'scrapfly/estarland' }; const modPath = MODULES[id]; if (!modPath) throw new Error(`unknown connector ${id}`); const meta = localMeta(JSON.parse(readFileSync(path.join(connectorsDir, modPath, 'meta.json'), 'utf8'))); const mod = (await import(path.join(connectorsDir, modPath, 'index.ts'))) as { default: (m: typeof meta) => { crawl: (ctx: never) => AsyncIterable; normalize: (r: never) => Promise } }; const connector = mod.default(meta); const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY }); const limit = Number(limitArg ?? 1) || 1; const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit, ...(seeds.length ? { seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) }); /** Keep only `n` items of the listy part of a payload, and cut a snapshot to the first `keep` element blocks. */ function trim(payload: Record, snapshot: string | null | undefined): Record { const p = { ...payload }; for (const key of ['issues', 'tiles', 'boxes', 'items', 'cards']) { if (Array.isArray(p[key])) p[key] = (p[key] as unknown[]).slice(0, id === 'mycomicshop' ? 6 : 30); } if (snapshot) { const marker = id === 'mycomicshop' ? '

' : id === 'entertainment-earth' ? '

273' : id === 'estarland' ? snapshot.slice(snapshot.indexOf('

'), snapshot.indexOf('

') + 1500) : ''; p.snapshot = `${(snapshot.match(/<title>([^<]*)/)?.[1] ?? '').trim()}${head}${blocks.join('\n')}`; } } return p; } let n = 0; const urlSeeds = seeds.filter((s) => /^https?:\/\//.test(s)); const lookup = (connector as unknown as { lookup?: (url: string, ctx: unknown) => Promise }).lookup; async function* source(): AsyncIterable { if (urlSeeds.length && lookup) { for (const u of urlSeeds) yield* await lookup.call(connector, u, ctx); return; } yield* connector.crawl(ctx as never); } for await (const raw of source()) { const payload = trim((raw.payload ?? {}) as Record, raw.snapshot); const normalized = await connector.normalize({ url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() } as never); const name = `${id === 'videogametrader' || id === 'mattel-creations' ? String((raw.payload as { collection?: string }).collection ?? 'shop') + '-' : ''}${String(raw.externalId ?? n).replace(/[^A-Za-z0-9]+/g, '-').replace(/^-|-$/g, '')}`; console.log(`[${id}] ${raw.url} → externalId=${raw.externalId} normalized=${normalized.length}`); saveFixture(id, name, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() }, expect: { minCount: normalized.length ? 1 : 0, kinds: id === 'mycomicshop' ? ['listing', 'auction_lot'] : [raw.kind] }, note: `Live capture of ${raw.url} on ${new Date().toISOString().slice(0, 10)} (payload lists trimmed; snapshot cut to a few blocks).`, }); n++; } console.log(`[${id}] fixtures=${n} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`);