TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { getGrader, normalizeCondition } from '@rareindex/taxonomy';4import { parsePrice, type AssetAttributes, type NormalizedRecord, type NormalizedSale } from '@rareindex/shared';56/**7 * ComicConnect sold archive — realised prices for comic books (auctions + fixed-price sales).8 * One raw record per browse page (compact parsed cards); normalise → one sale per card.9 */1011const BASE = 'https://www.comicconnect.com';12const PARSER_VERSION = '1.0.0';1314export const CardSchema = z.object({15 itemId: z.string(),16 url: z.string(),17 title: z.string(),18 gradeLine: z.string(),19 soldOn: z.string(),20 soldFor: z.number(),21 saleType: z.enum(['auction', 'sale', 'unknown']),22 buyersPremium: z.boolean(),23 noReserve: z.boolean(),24 comments: z.string().nullable(),25 image: z.string().nullable(),26});27export const PagePayloadSchema = z.object({ kind: z.literal('sold_page'), url: z.string(), page: z.number(), totalResults: z.number().nullable(), cards: z.array(CardSchema) });28export type PagePayload = z.infer<typeof PagePayloadSchema>;2930export function parseSoldPage(htmlText: string, url: string, page: number): PagePayload {31 const $ = H.load(htmlText);32 const total = $('.table_info').first().text().match(/([\d,]+)\s+Results/)?.[1] ?? null;33 const cards: z.infer<typeof CardSchema>[] = [];34 $('.itempreview').each((_, el) => {35 const $el = $(el);36 const href = $el.find('.titleline').closest('a').attr('href') ?? $el.find('.mainimg a').attr('href') ?? '';37 const itemId = href.match(/\/item\/(\d+)/)?.[1];38 const title = H.text($el.find('.titleline'));39 const gradeLine = H.text($el.find('.grade')) ?? '';40 const soldOn = (H.text($el.find('.endednotice')) ?? '').replace(/^Sold on\s*/i, '');41 const priceTxt = H.text($el.find('.pricing .val.prc'));42 const price = parsePrice(priceTxt, 'USD');43 if (!itemId || !title || !soldOn || !price || price.amount <= 0) return;44 const dataType = $el.attr('data-type') ?? '';45 const comments = $el.find('.comments').clone().find('.ctip').remove().end();46 comments.find('b').remove();47 const commentText = H.text(comments);48 const img = $el.find('.mainimg img').attr('src') ?? null;49 cards.push({50 itemId,51 url: `${BASE}/item/${itemId}`,52 title,53 gradeLine,54 soldOn,55 soldFor: price.amount,56 saleType: dataType === 'auction' ? 'auction' : dataType === 'sale' || dataType === 'buynow' ? 'sale' : 'unknown',57 buyersPremium: $el.attr('bp') === 'true',58 noReserve: /no reserve/i.test($el.find('.reserve').text()),59 comments: commentText,60 image: img ? (img.startsWith('http') ? img : BASE + img) : null,61 });62 });63 return { kind: 'sold_page', url, page, totalResults: total ? Number(total.replace(/,/g, '')) : null, cards };64}6566/** "X-MEN (1963-2011) #282" → { series: 'X-Men', seriesYears: '1963-2011', issue: '282', year: 1963 when single year } */67export function parseComicTitle(title: string): { series: string; seriesYears: string | null; issue: string | null; year: number | null; groupLot: boolean } {68 const groupLot = /group lot/i.test(title);69 let t = title.replace(/\s+Comic Book Group Lot$/i, '').trim();70 const issue = t.match(/#\s*([0-9]+[A-Za-z]?(?:\.[0-9]+)?(?:\/[0-9]+)?)/)?.[1] ?? null;71 const yearsM = t.match(/\((\d{4})(?:-(\d{2,4}))?\)/);72 const seriesYears = yearsM ? yearsM[0].slice(1, -1) : null;73 const year = yearsM && !yearsM[2] ? Number(yearsM[1]) : null;74 if (yearsM) t = t.replace(yearsM[0], ' ');75 if (issue) t = t.replace(/#\s*[0-9]+[A-Za-z]?(?:\.[0-9]+)?(?:\/[0-9]+)?/, ' ');76 const series = t77 .replace(/\s+/g, ' ')78 .trim()79 .toLowerCase()80 .replace(/(^|[\s(/-])([a-z])/g, (m, pre: string, c: string) => pre + c.toUpperCase());81 return { series, seriesYears, issue, year, groupLot };82}8384/** "Marvel CGC NM/M: 9.8" → { publisher: 'Marvel', grader: 'cgc', grade: '9.8', label: 'NM/M' } ; "Marvel VF/NM: 9.0" → raw grade */85const GRADE_LABEL = '(?:GEM MT|GEM|MT|NM/M|NM\\+|NM-|NM|VF/NM|VF\\+|VF-|VF|FN/VF|FN\\+|FN-|FN|VG/FN|VG\\+|VG-|VG|GD/VG|GD\\+|GD-|GD|FR/GD|FR|PR|M)';86const GRADE_LINE_RE = new RegExp(`^(.*?)\\s*(?:\\b(CGC|CBCS|PGX|EGS)\\b\\s*)?(${GRADE_LABEL})?\\s*:\\s*([\\d.]+)\\s*$`);8788export function parseGradeLine(line: string): { publisher: string | null; grader: string | null; grade: string | null; label: string | null } {89 const m = line.trim().match(GRADE_LINE_RE);90 if (!m) return { publisher: line || null, grader: null, grade: null, label: null };91 const publisher = m[1]?.trim() || null;92 const grader = m[2] ? (getGrader(m[2])?.slug ?? m[2].toLowerCase()) : 'raw';93 return { publisher, grader, grade: m[4] ?? null, label: m[3]?.trim() ?? null };94}9596function publisherCategory(publisher: string | null): string {97 const p = (publisher ?? '').toLowerCase();98 if (/marvel|timely|atlas/.test(p)) return 'marvel_comics';99 if (/^dc\b|dc comics|vertigo|wildstorm|national/.test(p)) return 'dc_comics';100 return 'independent_comics';101}102103export class ComicConnectConnector extends BaseConnector {104 readonly version = '1.0.0';105 readonly parserVersion = PARSER_VERSION;106 protected override minIntervalMs = 1500;107108 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {109 const browsePath = String(this.meta.config.browsePath ?? '/browse/comics/');110 const pages = Number(this.meta.config.pagesPerRun ?? 25);111 const sort = String(this.meta.config.sortType ?? 'ended_desc');112 const start = ctx.options.mode === 'backfill' ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;113 let count = 0;114 for (let page = start; page < start + pages; page++) {115 if (ctx.signal?.aborted || this.reached(ctx, count)) break;116 const url = `${BASE}${browsePath}?filtertype=Sold&sort_type=${encodeURIComponent(sort)}&page=${page}`;117 await this.throttle();118 const res = await ctx.fetch(url, {119 responseType: 'text',120 expect: ['title', 'price', 'date', 'status'],121 parse: (r) => {122 if (!r.html) return null;123 const p = parseSoldPage(r.html, url, page);124 const c = p.cards[0];125 return { title: c?.title ?? null, price: c?.soldFor ?? null, date: c?.soldOn ?? null, status: c ? 'sold' : null };126 },127 });128 if (!res.success || !res.html) {129 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);130 break;131 }132 const payload = parseSoldPage(res.html, url, page);133 if (payload.cards.length === 0) {134 ctx.anomaly('empty_page', url);135 break;136 }137 count++;138 if (ctx.options.mode === 'backfill') await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });139 yield { url, externalId: `sold:${browsePath}:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };140 }141 }142143 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {144 const p = PagePayloadSchema.parse(raw.payload);145 const out: NormalizedSale[] = [];146 for (const c of p.cards) {147 const t = parseComicTitle(c.title);148 const g = parseGradeLine(c.gradeLine);149 // "Friday, 08/14/2026 12:15 AM" → US local (Eastern) — we keep the date portion as UTC midnight to avoid faking precision.150 const dm = c.soldOn.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})/);151 if (!dm) continue;152 const saleDate = new Date(Date.UTC(Number(dm[3]), Number(dm[1]) - 1, Number(dm[2])));153 const categorySlug = publisherCategory(g.publisher);154 const rawCondition = g.grader === 'raw' && g.label ? g.label : null;155 const attributes: AssetAttributes = {156 categorySlug,157 subcategorySlug: null,158 franchise: null,159 brand: g.publisher,160 series: t.seriesYears ? `${t.series} (${t.seriesYears})` : t.series,161 set: t.series,162 setCode: null,163 name: t.series,164 model: null,165 reference: null,166 number: t.issue,167 year: t.year,168 edition: null,169 variant: null,170 language: 'English',171 region: null,172 country: 'US',173 material: null,174 size: null,175 color: null,176 rarity: null,177 productionQuantity: null,178 originalMsrp: null,179 originalMsrpCurrency: null,180 identifiers: { comicconnect_item: c.itemId },181 metadata: { series_years: t.seriesYears, buyers_premium_applies: c.buyersPremium, no_reserve: c.noReserve, sold_on_raw: c.soldOn, grade_label: g.label },182 };183 out.push({184 kind: 'sale',185 connectorId: this.meta.id,186 sourceId: this.meta.sourceId,187 sourceUrl: c.url,188 externalId: c.itemId,189 rawTitle: `${c.title} ${c.gradeLine}`.trim(),190 description: c.comments,191 imageUrls: c.image ? [c.image] : [],192 attributes,193 grade: { grader: g.grader === 'raw' ? 'raw' : g.grader, grade: g.grader === 'raw' ? null : g.grade, qualifier: null, certificationNumber: null },194 condition: { condition: normalizeCondition(categorySlug, rawCondition ?? (g.grader === 'raw' ? g.label : null)), conditionRaw: g.grader === 'raw' ? `${g.label ?? ''} ${g.grade ?? ''}`.trim() || null : null, completeness: null },195 observedAt: raw.fetchedAt,196 confidence: 0.9,197 parserVersion: PARSER_VERSION,198 saleType: c.saleType === 'auction' ? 'auction' : c.saleType === 'sale' ? 'fixed_price' : 'unknown',199 saleDate,200 price: c.soldFor,201 currency: 'USD',202 buyerPremiumIncluded: false,203 quantity: 1,204 isBundle: t.groupLot,205 location: 'US',206 auctionHouse: 'ComicConnect',207 lotNumber: null,208 });209 }210 return out;211 }212}213214export default (meta: ConnectorMeta) => new ComicConnectConnector(meta);215