import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { getGrader, normalizeCondition } from '@rareindex/taxonomy'; import { parsePrice, type AssetAttributes, type NormalizedRecord, type NormalizedSale } from '@rareindex/shared'; /** * ComicConnect sold archive — realised prices for comic books (auctions + fixed-price sales). * One raw record per browse page (compact parsed cards); normalise → one sale per card. */ const BASE = 'https://www.comicconnect.com'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ itemId: z.string(), url: z.string(), title: z.string(), gradeLine: z.string(), soldOn: z.string(), soldFor: z.number(), saleType: z.enum(['auction', 'sale', 'unknown']), buyersPremium: z.boolean(), noReserve: z.boolean(), comments: z.string().nullable(), image: z.string().nullable(), }); export const PagePayloadSchema = z.object({ kind: z.literal('sold_page'), url: z.string(), page: z.number(), totalResults: z.number().nullable(), cards: z.array(CardSchema) }); export type PagePayload = z.infer; export function parseSoldPage(htmlText: string, url: string, page: number): PagePayload { const $ = H.load(htmlText); const total = $('.table_info').first().text().match(/([\d,]+)\s+Results/)?.[1] ?? null; const cards: z.infer[] = []; $('.itempreview').each((_, el) => { const $el = $(el); const href = $el.find('.titleline').closest('a').attr('href') ?? $el.find('.mainimg a').attr('href') ?? ''; const itemId = href.match(/\/item\/(\d+)/)?.[1]; const title = H.text($el.find('.titleline')); const gradeLine = H.text($el.find('.grade')) ?? ''; const soldOn = (H.text($el.find('.endednotice')) ?? '').replace(/^Sold on\s*/i, ''); const priceTxt = H.text($el.find('.pricing .val.prc')); const price = parsePrice(priceTxt, 'USD'); if (!itemId || !title || !soldOn || !price || price.amount <= 0) return; const dataType = $el.attr('data-type') ?? ''; const comments = $el.find('.comments').clone().find('.ctip').remove().end(); comments.find('b').remove(); const commentText = H.text(comments); const img = $el.find('.mainimg img').attr('src') ?? null; cards.push({ itemId, url: `${BASE}/item/${itemId}`, title, gradeLine, soldOn, soldFor: price.amount, saleType: dataType === 'auction' ? 'auction' : dataType === 'sale' || dataType === 'buynow' ? 'sale' : 'unknown', buyersPremium: $el.attr('bp') === 'true', noReserve: /no reserve/i.test($el.find('.reserve').text()), comments: commentText, image: img ? (img.startsWith('http') ? img : BASE + img) : null, }); }); return { kind: 'sold_page', url, page, totalResults: total ? Number(total.replace(/,/g, '')) : null, cards }; } /** "X-MEN (1963-2011) #282" → { series: 'X-Men', seriesYears: '1963-2011', issue: '282', year: 1963 when single year } */ export function parseComicTitle(title: string): { series: string; seriesYears: string | null; issue: string | null; year: number | null; groupLot: boolean } { const groupLot = /group lot/i.test(title); let t = title.replace(/\s+Comic Book Group Lot$/i, '').trim(); const issue = t.match(/#\s*([0-9]+[A-Za-z]?(?:\.[0-9]+)?(?:\/[0-9]+)?)/)?.[1] ?? null; const yearsM = t.match(/\((\d{4})(?:-(\d{2,4}))?\)/); const seriesYears = yearsM ? yearsM[0].slice(1, -1) : null; const year = yearsM && !yearsM[2] ? Number(yearsM[1]) : null; if (yearsM) t = t.replace(yearsM[0], ' '); if (issue) t = t.replace(/#\s*[0-9]+[A-Za-z]?(?:\.[0-9]+)?(?:\/[0-9]+)?/, ' '); const series = t .replace(/\s+/g, ' ') .trim() .toLowerCase() .replace(/(^|[\s(/-])([a-z])/g, (m, pre: string, c: string) => pre + c.toUpperCase()); return { series, seriesYears, issue, year, groupLot }; } /** "Marvel CGC NM/M: 9.8" → { publisher: 'Marvel', grader: 'cgc', grade: '9.8', label: 'NM/M' } ; "Marvel VF/NM: 9.0" → raw grade */ const GRADE_LABEL = '(?:GEM MT|GEM|MT|NM/M|NM\\+|NM-|NM|VF/NM|VF\\+|VF-|VF|FN/VF|FN\\+|FN-|FN|VG/FN|VG\\+|VG-|VG|GD/VG|GD\\+|GD-|GD|FR/GD|FR|PR|M)'; const GRADE_LINE_RE = new RegExp(`^(.*?)\\s*(?:\\b(CGC|CBCS|PGX|EGS)\\b\\s*)?(${GRADE_LABEL})?\\s*:\\s*([\\d.]+)\\s*$`); export function parseGradeLine(line: string): { publisher: string | null; grader: string | null; grade: string | null; label: string | null } { const m = line.trim().match(GRADE_LINE_RE); if (!m) return { publisher: line || null, grader: null, grade: null, label: null }; const publisher = m[1]?.trim() || null; const grader = m[2] ? (getGrader(m[2])?.slug ?? m[2].toLowerCase()) : 'raw'; return { publisher, grader, grade: m[4] ?? null, label: m[3]?.trim() ?? null }; } function publisherCategory(publisher: string | null): string { const p = (publisher ?? '').toLowerCase(); if (/marvel|timely|atlas/.test(p)) return 'marvel_comics'; if (/^dc\b|dc comics|vertigo|wildstorm|national/.test(p)) return 'dc_comics'; return 'independent_comics'; } export class ComicConnectConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; async *crawl(ctx: CrawlContext): AsyncIterable { const browsePath = String(this.meta.config.browsePath ?? '/browse/comics/'); const pages = Number(this.meta.config.pagesPerRun ?? 25); const sort = String(this.meta.config.sortType ?? 'ended_desc'); const start = ctx.options.mode === 'backfill' ? Number(ctx.options.cursor?.nextPage ?? 1) : 1; let count = 0; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}${browsePath}?filtertype=Sold&sort_type=${encodeURIComponent(sort)}&page=${page}`; await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'date', 'status'], parse: (r) => { if (!r.html) return null; const p = parseSoldPage(r.html, url, page); const c = p.cards[0]; return { title: c?.title ?? null, price: c?.soldFor ?? null, date: c?.soldOn ?? null, status: c ? 'sold' : null }; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseSoldPage(res.html, url, page); if (payload.cards.length === 0) { ctx.anomaly('empty_page', url); break; } count++; if (ctx.options.mode === 'backfill') await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() }); yield { url, externalId: `sold:${browsePath}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedSale[] = []; for (const c of p.cards) { const t = parseComicTitle(c.title); const g = parseGradeLine(c.gradeLine); // "Friday, 08/14/2026 12:15 AM" → US local (Eastern) — we keep the date portion as UTC midnight to avoid faking precision. const dm = c.soldOn.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})/); if (!dm) continue; const saleDate = new Date(Date.UTC(Number(dm[3]), Number(dm[1]) - 1, Number(dm[2]))); const categorySlug = publisherCategory(g.publisher); const rawCondition = g.grader === 'raw' && g.label ? g.label : null; const attributes: AssetAttributes = { categorySlug, subcategorySlug: null, franchise: null, brand: g.publisher, series: t.seriesYears ? `${t.series} (${t.seriesYears})` : t.series, set: t.series, setCode: null, name: t.series, model: null, reference: null, number: t.issue, year: t.year, edition: null, variant: null, language: 'English', region: null, country: 'US', material: null, size: null, color: null, rarity: null, productionQuantity: null, originalMsrp: null, originalMsrpCurrency: null, identifiers: { comicconnect_item: c.itemId }, metadata: { series_years: t.seriesYears, buyers_premium_applies: c.buyersPremium, no_reserve: c.noReserve, sold_on_raw: c.soldOn, grade_label: g.label }, }; out.push({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, externalId: c.itemId, rawTitle: `${c.title} ${c.gradeLine}`.trim(), description: c.comments, imageUrls: c.image ? [c.image] : [], attributes, grade: { grader: g.grader === 'raw' ? 'raw' : g.grader, grade: g.grader === 'raw' ? null : g.grade, qualifier: null, certificationNumber: null }, condition: { condition: normalizeCondition(categorySlug, rawCondition ?? (g.grader === 'raw' ? g.label : null)), conditionRaw: g.grader === 'raw' ? `${g.label ?? ''} ${g.grade ?? ''}`.trim() || null : null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, saleType: c.saleType === 'auction' ? 'auction' : c.saleType === 'sale' ? 'fixed_price' : 'unknown', saleDate, price: c.soldFor, currency: 'USD', buyerPremiumIncluded: false, quantity: 1, isBundle: t.groupLot, location: 'US', auctionHouse: 'ComicConnect', lotNumber: null, }); } return out; } } export default (meta: ConnectorMeta) => new ComicConnectConnector(meta);