TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../../api/_lib/shared.js';5import { cardCategorySlug, money } from '../../api/_lib/wave4.js';67/** COMC category browse pages (Firecrawl markdown) → card listings with grades and asking prices. */8const PARSER_VERSION = '1.0.0';9const SITE = 'https://www.comc.com';1011const ListingSchema = z.object({ title: z.string(), url: z.string(), image: z.string().nullable(), setLine: z.string().nullable(), price: z.number() });12const RawPayloadSchema = z.object({ seed: z.string(), page: z.number(), listings: z.array(ListingSchema) });13export type ComcPayload = z.infer<typeof RawPayloadSchema>;1415/** Parse a rendered browse page. Exported for tests. */16export function parseBrowseMarkdown(md: string): z.infer<typeof ListingSchema>[] {17 const lines = md.split('\n').map((l) => l.trim());18 const out: z.infer<typeof ListingSchema>[] = [];19 let setLine: string | null = null;20 let image: string | null = null;21 for (let i = 0; i < lines.length; i++) {22 const l = lines[i]!;23 const img = l.match(/^\[!\[[^\]]*\]\((https?:\/\/img\.comc\.com[^)\s]+)\)\]\(https?:\/\/www\.comc\.com\/Cards\/[^)]+\)$/);24 if (img) {25 image = img[1]!;26 continue;27 }28 if (/^\d{4}(?:-\d{2})?\s.+#\S+$/.test(l.replace(/\\/g, ''))) {29 setLine = l.replace(/\\/g, '');30 continue;31 }32 const h = l.match(/^###\s+\[(.+?)\]\((https?:\/\/www\.comc\.com\/Cards\/[^)\s]+)\)\s*$/);33 if (!h) continue;34 let price: number | null = null;35 for (let j = i + 1; j < Math.min(lines.length, i + 5); j++) {36 const m = lines[j]!.match(/^\$([\d,]+\.\d{2})$/);37 if (m) {38 price = money(m[1]);39 break;40 }41 if (lines[j]!.startsWith('###')) break;42 }43 if (price === null) continue;44 out.push({ title: h[1]!.replace(/\\/g, ''), url: h[2]!, image, setLine, price });45 image = null;46 }47 return out;48}4950const GRADE_RE = /\[(PSA|BGS|CSG|CGC|SGC|TAG|HGA|ACE|BVG|GMA|ISA|KSA|MNT|BCCG|PSA\/DNA|BAS|JSA)\s*(\d{1,2}(?:\.\d)?)?\s*([^\]]*)\]/i;5152export function parseComcTitle(title: string): { name: string; grader: string | null; grade: string | null; qualifier: string | null; auto: boolean } {53 const m = title.match(GRADE_RE);54 const name = title.replace(/\s*\[[^\]]*\]\s*/g, ' ').replace(/\s+/g, ' ').trim();55 if (!m) return { name, grader: null, grade: null, qualifier: null, auto: /\bauto/i.test(title) };56 const grader = m[1]!.toLowerCase().replace('/', '_');57 const grade = m[2] ?? null;58 const qualifier = m[3]?.trim() || null;59 const isAuth = /dna|bas|jsa/.test(grader);60 return { name, grader: isAuth && !grade ? null : grader, grade, qualifier: isAuth ? `${m[1]} ${qualifier ?? ''}`.trim() : qualifier, auto: /\bauto/i.test(title) };61}6263export function parseSetLine(setLine: string | null): { year: number | null; set: string | null; variant: string | null; number: string | null } {64 if (!setLine) return { year: null, set: null, variant: null, number: null };65 const m = setLine.match(/^(\d{4})(?:-\d{2})?\s+(.+?)(?:\s+-\s+\[(.+?)\])?\s+#(\S+)$/);66 if (!m) return { year: null, set: setLine, variant: null, number: null };67 const year = Number(m[1]);68 const variant = m[3] && !/^base$/i.test(m[3]) ? m[3] : null;69 return { year, set: `${m[1]} ${m[2]}`.trim(), variant, number: m[4] ?? null };70}7172function seedCategory(seed: string): string | null {73 const sport = seed.split('/')[1]?.replace(/_/g, ' ') ?? '';74 return cardCategorySlug(sport) ?? (/^racing$/i.test(sport) ? 'other_sports_cards' : null);75}7677export class ComcConnector extends BaseConnector {78 readonly version = '1.0.0';79 readonly parserVersion = PARSER_VERSION;80 protected override minIntervalMs = 2000;8182 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {83 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []);84 const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);85 let count = 0;86 for (const seed of seeds) {87 for (let page = 1; page <= pages; page++) {88 if (ctx.signal?.aborted) return;89 if (this.reached(ctx, count)) return;90 const url = `${SITE}/${seed.replace(/^\/+/, '')},sh${page > 1 ? `,p${page}` : ''}`;91 await this.throttle();92 const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 3000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.includes('Listings') ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });93 if (!res.success || !res.markdown) {94 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);95 break;96 }97 const listings = parseBrowseMarkdown(res.markdown);98 if (!listings.length) {99 ctx.anomaly('parse_failure_page', url);100 break;101 }102 count++;103 const payload: ComcPayload = { seed, page, listings };104 yield { url, externalId: `${seed}:p${page}`, kind: 'listing', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };105 const total = res.markdown.match(/of\s+\*\*([\d,]+)\*\*/)?.[1];106 if (total && Number(total.replace(/,/g, '')) <= page * 100) break;107 }108 await ctx.setCursor({ seed, at: new Date().toISOString() });109 }110 }111112 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {113 const p = RawPayloadSchema.parse(raw.payload);114 const categorySlug = seedCategory(p.seed);115 if (!categorySlug) return [];116 const out: NormalizedRecord[] = [];117 for (const l of p.listings) {118 const t = parseComcTitle(l.title);119 const s = parseSetLine(l.setLine);120 const externalId = l.url.replace(`${SITE}/Cards/`, '').replace(/\/+$/, '');121 const a = attrs({ categorySlug, name: t.name, set: s.set, year: s.year, number: s.number, variant: s.variant, identifiers: { comc_path: externalId }, metadata: { autograph: t.auto } });122 out.push(123 NormalizedListingSchema.parse({124 kind: 'listing',125 connectorId: this.meta.id,126 sourceId: this.meta.sourceId,127 sourceUrl: l.url,128 externalId,129 rawTitle: l.setLine ? `${l.setLine} ${l.title}` : l.title,130 imageUrls: l.image ? [l.image.replace(/size=biggerthumb/, 'size=large')] : [],131 attributes: a,132 grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier },133 condition: {},134 observedAt: raw.fetchedAt,135 confidence: 0.8,136 parserVersion: PARSER_VERSION,137 listingType: 'fixed_price',138 price: l.price,139 currency: 'USD',140 seller: 'COMC consignment',141 availability: 'available',142 }),143 );144 }145 return out;146 }147}148149export default (meta: ConnectorMeta) => new ComcConnector(meta);150