TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Live fixture capture for comiclink (Scrapfly, ~2 credits): schedule page + newest comics session results.3 * Usage: set -a; . ./.env; set +a; pnpm tsx connectors/scrapfly/comiclink/_capture.ts [sessionCode]4 */5import { createCrawlContext, createRouter } from '@rareindex/connectors';6import { saveFixture } from '@rareindex/connectors/testing';7import { childLogger } from '@rareindex/shared';8import { localMeta } from '../../api/_lib/local-meta.js';9import { loadEnv, useGroupDomains } from '../../api/_g6-comics-toys-games-lib/capture-env.js';10import meta from './meta.json' with { type: 'json' };11import createConnector, { type PagePayload } from './index.js';1213loadEnv();14useGroupDomains();15const connector = createConnector(localMeta(meta));16const router = createRouter({ scrapflyApiKey: process.env.SCRAPFLY_API_KEY });17const seeds = process.argv.slice(2);18const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 1, ...(seeds.length ? { seeds } : {}) }, log: childLogger({ connector: 'comiclink', level: 'warn' }) });1920for await (const raw of connector.crawl(ctx)) {21 const p = raw.payload as PagePayload;22 const html = raw.snapshot ?? '';23 // Trim the snapshot to the first ~6 result blocks so parser tests stay small but real.24 const start = html.indexOf('<div class="bookcover"');25 const blocks = html.slice(start).split('<div class="bookcover"').slice(1, 7).map((b) => '<div class="bookcover"' + b);26 const trimmed = `<html><body>${blocks.join('\n')}</body></html>`;27 const items = p.items.slice(0, 40);28 console.log(`session ${p.session.code} "${p.session.label}" end=${p.session.endYear}-${p.session.endMonth} items=${p.items.length} first=${JSON.stringify(items[0])}`);29 saveFixture('comiclink', `results-${p.session.code.replace(/[^A-Za-z0-9-]/g, '_')}`, {30 raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: { ...p, items, snapshot: trimmed }, fetchedAt: raw.fetchedAt ?? new Date() },31 expect: { minCount: 10, kinds: ['sale'], requiredFields: ['attributes.series', 'attributes.number', 'saleDate', 'price'] },32 note: `Live Scrapfly capture of ${raw.url} (session label from /auctions/auctionschedule.asp); items trimmed to 40, snapshot trimmed to 6 blocks.`,33 });34}35console.log('engineStats=', JSON.stringify(ctx.engineStats), 'anomalies=', ctx.anomalies);36