import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { attrs } from '../../api/_lib/shared.js'; import { cardCategorySlug, money } from '../../api/_lib/wave4.js'; /** COMC category browse pages (Firecrawl markdown) → card listings with grades and asking prices. */ const PARSER_VERSION = '1.0.0'; const SITE = 'https://www.comc.com'; const ListingSchema = z.object({ title: z.string(), url: z.string(), image: z.string().nullable(), setLine: z.string().nullable(), price: z.number() }); const RawPayloadSchema = z.object({ seed: z.string(), page: z.number(), listings: z.array(ListingSchema) }); export type ComcPayload = z.infer; /** Parse a rendered browse page. Exported for tests. */ export function parseBrowseMarkdown(md: string): z.infer[] { const lines = md.split('\n').map((l) => l.trim()); const out: z.infer[] = []; let setLine: string | null = null; let image: string | null = null; for (let i = 0; i < lines.length; i++) { const l = lines[i]!; const img = l.match(/^\[!\[[^\]]*\]\((https?:\/\/img\.comc\.com[^)\s]+)\)\]\(https?:\/\/www\.comc\.com\/Cards\/[^)]+\)$/); if (img) { image = img[1]!; continue; } if (/^\d{4}(?:-\d{2})?\s.+#\S+$/.test(l.replace(/\\/g, ''))) { setLine = l.replace(/\\/g, ''); continue; } const h = l.match(/^###\s+\[(.+?)\]\((https?:\/\/www\.comc\.com\/Cards\/[^)\s]+)\)\s*$/); if (!h) continue; let price: number | null = null; for (let j = i + 1; j < Math.min(lines.length, i + 5); j++) { const m = lines[j]!.match(/^\$([\d,]+\.\d{2})$/); if (m) { price = money(m[1]); break; } if (lines[j]!.startsWith('###')) break; } if (price === null) continue; out.push({ title: h[1]!.replace(/\\/g, ''), url: h[2]!, image, setLine, price }); image = null; } return out; } const GRADE_RE = /\[(PSA|BGS|CSG|CGC|SGC|TAG|HGA|ACE|BVG|GMA|ISA|KSA|MNT|BCCG|PSA\/DNA|BAS|JSA)\s*(\d{1,2}(?:\.\d)?)?\s*([^\]]*)\]/i; export function parseComcTitle(title: string): { name: string; grader: string | null; grade: string | null; qualifier: string | null; auto: boolean } { const m = title.match(GRADE_RE); const name = title.replace(/\s*\[[^\]]*\]\s*/g, ' ').replace(/\s+/g, ' ').trim(); if (!m) return { name, grader: null, grade: null, qualifier: null, auto: /\bauto/i.test(title) }; const grader = m[1]!.toLowerCase().replace('/', '_'); const grade = m[2] ?? null; const qualifier = m[3]?.trim() || null; const isAuth = /dna|bas|jsa/.test(grader); return { name, grader: isAuth && !grade ? null : grader, grade, qualifier: isAuth ? `${m[1]} ${qualifier ?? ''}`.trim() : qualifier, auto: /\bauto/i.test(title) }; } export function parseSetLine(setLine: string | null): { year: number | null; set: string | null; variant: string | null; number: string | null } { if (!setLine) return { year: null, set: null, variant: null, number: null }; const m = setLine.match(/^(\d{4})(?:-\d{2})?\s+(.+?)(?:\s+-\s+\[(.+?)\])?\s+#(\S+)$/); if (!m) return { year: null, set: setLine, variant: null, number: null }; const year = Number(m[1]); const variant = m[3] && !/^base$/i.test(m[3]) ? m[3] : null; return { year, set: `${m[1]} ${m[2]}`.trim(), variant, number: m[4] ?? null }; } function seedCategory(seed: string): string | null { const sport = seed.split('/')[1]?.replace(/_/g, ' ') ?? ''; return cardCategorySlug(sport) ?? (/^racing$/i.test(sport) ? 'other_sports_cards' : null); } export class ComcConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []); const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2); let count = 0; for (const seed of seeds) { for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted) return; if (this.reached(ctx, count)) return; const url = `${SITE}/${seed.replace(/^\/+/, '')},sh${page > 1 ? `,p${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 3000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.includes('Listings') ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) }); if (!res.success || !res.markdown) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const listings = parseBrowseMarkdown(res.markdown); if (!listings.length) { ctx.anomaly('parse_failure_page', url); break; } count++; const payload: ComcPayload = { seed, page, listings }; yield { url, externalId: `${seed}:p${page}`, kind: 'listing', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; const total = res.markdown.match(/of\s+\*\*([\d,]+)\*\*/)?.[1]; if (total && Number(total.replace(/,/g, '')) <= page * 100) break; } await ctx.setCursor({ seed, at: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = RawPayloadSchema.parse(raw.payload); const categorySlug = seedCategory(p.seed); if (!categorySlug) return []; const out: NormalizedRecord[] = []; for (const l of p.listings) { const t = parseComcTitle(l.title); const s = parseSetLine(l.setLine); const externalId = l.url.replace(`${SITE}/Cards/`, '').replace(/\/+$/, ''); const a = attrs({ categorySlug, name: t.name, set: s.set, year: s.year, number: s.number, variant: s.variant, identifiers: { comc_path: externalId }, metadata: { autograph: t.auto } }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.url, externalId, rawTitle: l.setLine ? `${l.setLine} ${l.title}` : l.title, imageUrls: l.image ? [l.image.replace(/size=biggerthumb/, 'size=large')] : [], attributes: a, grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier }, condition: {}, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: l.price, currency: 'USD', seller: 'COMC consignment', availability: 'available', }), ); } return out; } } export default (meta: ConnectorMeta) => new ComcConnector(meta);