TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Live fixture capture for the g6 connectors (real pages/APIs, payloads trimmed, snapshots cut down).3 * Usage: pnpm tsx connectors/api/_g6-comics-toys-games-lib/capture.ts <connectorId> [limit] [seed...]4 * e.g. … capture.ts videogametrader 25 * … capture.ts mycomicshop 1 789916 * … capture.ts entertainment-earth 17 */8import { readFileSync } from 'node:fs';9import path from 'node:path';10import { fileURLToPath } from 'node:url';11import { createCrawlContext, createRouter, type RawRecordInput } from '@rareindex/connectors';12import { saveFixture } from '@rareindex/connectors/testing';13import { childLogger } from '@rareindex/shared';14import { localMeta } from '../_lib/local-meta.js';15import { loadEnv, useGroupDomains } from './capture-env.js';1617loadEnv();18useGroupDomains();19const here = path.dirname(fileURLToPath(import.meta.url));20const connectorsDir = path.resolve(here, '../..');21const [id, limitArg, ...seeds] = process.argv.slice(2);22if (!id) throw new Error('usage: capture.ts <connectorId> [limit] [seed...]');23const MODULES: Record<string, string> = { mycomicshop: 'scrapfly/mycomicshop', 'entertainment-earth': 'scrapfly/entertainment-earth', 'miniature-market': 'api/miniature-market', videogametrader: 'api/videogametrader', 'mattel-creations': 'api/mattel-creations', comiclink: 'scrapfly/comiclink', gcd: 'api/gcd', estarland: 'scrapfly/estarland' };24const modPath = MODULES[id];25if (!modPath) throw new Error(`unknown connector ${id}`);26const meta = localMeta(JSON.parse(readFileSync(path.join(connectorsDir, modPath, 'meta.json'), 'utf8')));27const mod = (await import(path.join(connectorsDir, modPath, 'index.ts'))) as { default: (m: typeof meta) => { crawl: (ctx: never) => AsyncIterable<RawRecordInput>; normalize: (r: never) => Promise<unknown[]> } };28const connector = mod.default(meta);29const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY, firecrawlApiKey: process.env.FIRECRAWL_API_KEY });30const limit = Number(limitArg ?? 1) || 1;31const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit, ...(seeds.length ? { seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });3233/** Keep only `n` items of the listy part of a payload, and cut a snapshot to the first `keep` element blocks. */34function trim(payload: Record<string, unknown>, snapshot: string | null | undefined): Record<string, unknown> {35 const p = { ...payload };36 for (const key of ['issues', 'tiles', 'boxes', 'items', 'cards']) {37 if (Array.isArray(p[key])) p[key] = (p[key] as unknown[]).slice(0, id === 'mycomicshop' ? 6 : 30);38 }39 if (snapshot) {40 const marker = id === 'mycomicshop' ? '<li class="issue">' : id === 'entertainment-earth' ? '<div class="grid-view item' : id === 'miniature-market' ? '<div class="card product-box' : id === 'estarland' ? '<a href="/product-description/' : null;41 if (marker) {42 const start = snapshot.indexOf(marker);43 const blocks = snapshot.slice(start).split(marker).slice(1).map((b) => marker + b).filter((b) => (id === 'estarland' ? b.includes('productConditionHolder') : true)).slice(0, id === 'mycomicshop' ? 2 : 4);44 const head = id === 'miniature-market' ? snapshot.slice(snapshot.indexOf('<nav aria-label="Pagination"'), snapshot.indexOf('<nav aria-label="Pagination"') + 600) : id === 'entertainment-earth' ? '<a href="?page=2">2</a><a href="?page=73">73</a>' : id === 'estarland' ? snapshot.slice(snapshot.indexOf('<div class="commingsoon_pagig">'), snapshot.indexOf('<div class="commingsoon_pagig">') + 1500) : '';45 p.snapshot = `<html><head><title>${(snapshot.match(/<title>([^<]*)/)?.[1] ?? '').trim()}</title></head><body>${head}${blocks.join('\n')}</body></html>`;46 }47 }48 return p;49}5051let n = 0;52const urlSeeds = seeds.filter((s) => /^https?:\/\//.test(s));53const lookup = (connector as unknown as { lookup?: (url: string, ctx: unknown) => Promise<RawRecordInput[]> }).lookup;54async function* source(): AsyncIterable<RawRecordInput> {55 if (urlSeeds.length && lookup) {56 for (const u of urlSeeds) yield* await lookup.call(connector, u, ctx);57 return;58 }59 yield* connector.crawl(ctx as never);60}61for await (const raw of source()) {62 const payload = trim((raw.payload ?? {}) as Record<string, unknown>, raw.snapshot);63 const normalized = await connector.normalize({ url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() } as never);64 const name = `${id === 'videogametrader' || id === 'mattel-creations' ? String((raw.payload as { collection?: string }).collection ?? 'shop') + '-' : ''}${String(raw.externalId ?? n).replace(/[^A-Za-z0-9]+/g, '-').replace(/^-|-$/g, '')}`;65 console.log(`[${id}] ${raw.url} → externalId=${raw.externalId} normalized=${normalized.length}`);66 saveFixture(id, name, {67 raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload, fetchedAt: raw.fetchedAt ?? new Date() },68 expect: { minCount: normalized.length ? 1 : 0, kinds: id === 'mycomicshop' ? ['listing', 'auction_lot'] : [raw.kind] },69 note: `Live capture of ${raw.url} on ${new Date().toISOString().slice(0, 10)} (payload lists trimmed; snapshot cut to a few blocks).`,70 });71 n++;72}73console.log(`[${id}] fixtures=${n} engineStats=${JSON.stringify(ctx.engineStats)} anomalies=${JSON.stringify(ctx.anomalies)}`);74