/** * Live fixture capture for comiclink (Scrapfly, ~2 credits): schedule page + newest comics session results. * Usage: set -a; . ./.env; set +a; pnpm tsx connectors/scrapfly/comiclink/_capture.ts [sessionCode] */ import { createCrawlContext, createRouter } from '@rareindex/connectors'; import { saveFixture } from '@rareindex/connectors/testing'; import { childLogger } from '@rareindex/shared'; import { localMeta } from '../../api/_lib/local-meta.js'; import { loadEnv, useGroupDomains } from '../../api/_g6-comics-toys-games-lib/capture-env.js'; import meta from './meta.json' with { type: 'json' }; import createConnector, { type PagePayload } from './index.js'; loadEnv(); useGroupDomains(); const connector = createConnector(localMeta(meta)); const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY }); const seeds = process.argv.slice(2); const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 1, ...(seeds.length ? { seeds } : {}) }, log: childLogger({ connector: 'comiclink', level: 'warn' }) }); for await (const raw of connector.crawl(ctx)) { const p = raw.payload as PagePayload; const html = raw.snapshot ?? ''; // Trim the snapshot to the first ~6 result blocks so parser tests stay small but real. const start = html.indexOf('

'

${blocks.join('\n')}`; const items = p.items.slice(0, 40); console.log(`session ${p.session.code} "${p.session.label}" end=${p.session.endYear}-${p.session.endMonth} items=${p.items.length} first=${JSON.stringify(items[0])}`); saveFixture('comiclink', `results-${p.session.code.replace(/[^A-Za-z0-9-]/g, '_')}`, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: { ...p, items, snapshot: trimmed }, fetchedAt: raw.fetchedAt ?? new Date() }, expect: { minCount: 10, kinds: ['sale'], requiredFields: ['attributes.series', 'attributes.number', 'saleDate', 'price'] }, note: `Live Scrapfly capture of ${raw.url} (session label from /auctions/auctionschedule.asp); items trimmed to 40, snapshot trimmed to 6 blocks.`, }); } console.log('engineStats=', JSON.stringify(ctx.engineStats), 'anomalies=', ctx.anomalies);