TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';4import { slugFromTitle } from '../../api/_auction-lib/categories.js';5import { attrs } from '../../api/_lib/shared.js';6import { parseComicGrade, parseComicTitle, sessionEndMonth, usd } from '../../api/_g6-comics-toys-games-lib/comics.js';78/**9 * ComicLink — public "Record-Setting Auction Results" pages (one per auction session since 2015):10 * /results/generate_highlights.asp?output=<code>. Each page lists ~300 sold books as11 * "TITLE #N / SOLD in CGC 9.4 NM / $1,365" with a link to the (login-gated) item page. Session12 * end month comes from the schedule page label ("Spring Featured: Comics (5-6/26)"), so sale dates13 * carry MONTH precision (metadata.sale_date_precision = "month"). Pages sit behind a Cloudflare14 * managed challenge → Scrapfly (no JS rendering, ~1 credit per page).15 */16const SITE = 'https://www.comiclink.com';17const SCHEDULE = `${SITE}/auctions/auctionschedule.asp`;18const PARSER_VERSION = '1.0.0';1920export const SessionSchema = z.object({ code: z.string(), label: z.string(), year: z.number().int().nullable(), endYear: z.number().int().nullable(), endMonth: z.number().int().nullable(), isArt: z.boolean() });21export type Session = z.infer<typeof SessionSchema>;22export const ResultItemSchema = z.object({ itemId: z.string().nullable(), title: z.string(), gradeLine: z.string().nullable(), priceText: z.string(), price: z.number(), image: z.string().nullable() });23export type ResultItem = z.infer<typeof ResultItemSchema>;24export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), url: z.string(), session: SessionSchema, items: z.array(ResultItemSchema), snapshot: z.string().optional() });25export type PagePayload = z.infer<typeof PagePayloadSchema>;2627/** Auction schedule → past sessions (newest first). Art sessions are flagged from code/label. */28export function parseSchedule(htmlText: string): Session[] {29 const $ = H.load(htmlText);30 const out: Session[] = [];31 const seen = new Set<string>();32 $('a[href*="generate_highlights.asp"]').each((_, el) => {33 const href = $(el).attr('href') ?? '';34 const code = decodeURIComponent(href.match(/output=([^&"']+)/)?.[1] ?? '').trim();35 if (!code || seen.has(code)) return;36 seen.add(code);37 const label = (H.text($(el)) ?? '').trim();38 const year = Number(code.match(/^(20\d{2})/)?.[1] ?? code.match(/\b(20\d{2})\b/)?.[1]) || null;39 const end = sessionEndMonth(label, year) ?? (year ? monthFromCode(code, year) : null);40 // Art-only sessions carry "Art"/"OA" in the code ("2019-11-Art-Auction", "2017-5-Comic-Art", "February 2017 OA");41 // mixed "Premium Comics and Art" sessions do not and are handled per item in normalize().42 const isArt = /art|\bOA\b/i.test(code);43 out.push({ code, label, year, endYear: end?.year ?? null, endMonth: end?.month ?? null, isArt });44 });45 out.sort((a, b) => (b.endYear ?? b.year ?? 0) * 100 + (b.endMonth ?? 0) - ((a.endYear ?? a.year ?? 0) * 100 + (a.endMonth ?? 0)));46 return out;47}4849/** "2019-11-ComicAuction" → {2019, 11}; "2023-1-Premium" → {2023, 1}. Season codes without a label month → null. */50export function monthFromCode(code: string, year: number): { year: number; month: number } | null {51 const m = code.match(/^20\d{2}-(\d{1,2})(?:-|[A-Za-z])/);52 if (m) {53 const month = Number(m[1]);54 if (month >= 1 && month <= 12) return { year, month };55 }56 return null;57}5859/** Results page → sold items. Blocks: <div class="bookcover"> … <a href="item.asp?id=N"><img src></a> … <b>TITLE<br>SOLD in CGC 9.4 NM<br>$1,365</b>. */60export function parseResultsPage(htmlText: string): ResultItem[] {61 const $ = H.load(htmlText);62 const out: ResultItem[] = [];63 $('div.bookcover').each((_, el) => {64 const $el = $(el);65 const href = $el.find('a[href*="item.asp"]').first().attr('href') ?? '';66 const itemId = href.match(/id=(\d+)/)?.[1] ?? null;67 const imgSrc = $el.find('img').first().attr('src') ?? null;68 const b = $el.find('b').last();69 const parts = (b.html() ?? '')70 .split(/<br\s*\/?>/i)71 .map((s) => H.load(`<x>${s}</x>`)('x').text().replace(/\s+/g, ' ').trim())72 .filter(Boolean);73 if (parts.length < 2) return;74 const priceText = parts.find((p) => /\$\s?[\d,]+/.test(p)) ?? '';75 const price = usd(priceText);76 const title = parts[0]!;77 const gradeLine = parts.find((p) => /^SOLD\b/i.test(p))?.replace(/^SOLD\s+(?:in|as)?\s*/i, '') ?? null;78 if (!title || !price) return;79 out.push({ itemId, title, gradeLine, priceText, price, image: imgSrc ? H.absUrl(SITE, imgSrc.replace(/^\/\.\//, '/')) : null });80 });81 return out;82}8384export class ComicLinkConnector extends BaseConnector {85 readonly version = '1.0.0';86 readonly parserVersion = PARSER_VERSION;87 protected override minIntervalMs = 3000;8889 private async fetchHtml(ctx: CrawlContext, url: string, expectItems: boolean): Promise<string | null> {90 await this.throttle(url);91 const res = await ctx.fetch(url, {92 engines: ['scrapfly'],93 renderJs: false,94 country: 'us',95 timeoutMs: 90_000,96 expect: expectItems ? ['title', 'price', 'status'] : ['title'],97 parse: (r) => {98 if (!r.html) return null;99 if (!expectItems) return { title: /generate_highlights/.test(r.html) ? 'ok' : null };100 const first = parseResultsPage(r.html)[0];101 return first ? { title: first.title, price: first.price, status: 'sold' } : null;102 },103 });104 if (!res.success || !res.html) {105 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);106 return null;107 }108 return res.html;109 }110111 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {112 const includeArt = Boolean(this.meta.config.includeArtSessions ?? false);113 const backfill = ctx.options.mode === 'backfill';114 const perRun = ctx.options.mode === 'probe' ? 1 : Number(backfill ? this.meta.config.backfillSessionsPerRun ?? 30 : this.meta.config.sessionsPerRun ?? 2);115 const scheduleHtml = await this.fetchHtml(ctx, SCHEDULE, false);116 if (!scheduleHtml) return;117 let sessions = parseSchedule(scheduleHtml).filter((s) => includeArt || !s.isArt);118 if (ctx.options.seeds?.length) sessions = sessions.filter((s) => ctx.options.seeds!.includes(s.code));119 if (!sessions.length) {120 ctx.anomaly('selector_missing', 'no result sessions found on the auction schedule page');121 return;122 }123 const cursor = (ctx.options.cursor ?? {}) as { index?: number; done?: boolean };124 let index = backfill ? Math.max(0, Number(cursor.index ?? 0)) : 0;125 let count = 0;126 for (let n = 0; n < perRun && index < sessions.length; n++, index++) {127 if (ctx.signal?.aborted || this.reached(ctx, count)) return;128 const s = sessions[index]!;129 const url = `${SITE}/results/generate_highlights.asp?output=${encodeURIComponent(s.code)}`;130 const page = await this.fetchHtml(ctx, url, true);131 if (!page) continue;132 const items = parseResultsPage(page);133 if (!items.length) {134 ctx.anomaly('parse_failure_page', url);135 continue;136 }137 if (!s.endMonth) ctx.anomaly('date_parse_failure', `${s.code}: no end month in label "${s.label}"`);138 const payload: PagePayload = { kind: 'results_page', url, session: s, items };139 count++;140 yield { url, externalId: `results:${s.code}`, kind: 'sale', engine: 'scrapfly', httpStatus: 200, payload, snapshot: page, fetchedAt: new Date() };141 if (backfill) {142 await ctx.setCursor({ index: index + 1, code: s.code, updatedAt: new Date().toISOString() });143 await ctx.progress({ page: index + 1, totalPages: sessions.length, itemsProcessed: count, reachedDate: s.endYear && s.endMonth ? new Date(Date.UTC(s.endYear, s.endMonth - 1, 1)) : null });144 }145 }146 if (backfill && index >= sessions.length) await ctx.setCursor({ done: true, index: 0, updatedAt: new Date().toISOString() });147 }148149 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {150 const p = PagePayloadSchema.parse(raw.payload);151 const s = p.session;152 if (!s.endYear || !s.endMonth) return []; // no source date → nothing we can honestly call a sale153 const saleDate = new Date(Date.UTC(s.endYear, s.endMonth - 1, 1));154 const out: NormalizedRecord[] = [];155 for (const it of p.items) {156 const t = parseComicTitle(it.title);157 const g = parseComicGrade(it.gradeLine ?? it.title);158 const artItem = s.isArt || /\b(original art|cover art|splash|page \d+|painting|sketch|illustration|commission)\b/i.test(it.title);159 const categorySlug = artItem ? 'art' : (slugFromTitle(it.title, 'comics') ?? 'independent_comics');160 const attributes = attrs({161 categorySlug,162 series: artItem ? null : t.series,163 set: artItem ? null : t.series,164 name: artItem ? it.title : `${t.series}${t.issue ? ` #${t.issue}` : ''}`,165 number: artItem ? null : t.issue,166 year: t.year,167 variant: t.variant,168 language: 'English',169 country: 'US',170 identifiers: it.itemId ? { comiclink_item_id: it.itemId } : {},171 metadata: { session_code: s.code, session_label: s.label, sale_date_precision: 'month', grade_line: it.gradeLine, grade_label: g.label, comic_art: artItem, highlights_page: true },172 });173 out.push(174 NormalizedSaleSchema.parse({175 kind: 'sale',176 connectorId: this.meta.id,177 sourceId: this.meta.sourceId,178 sourceUrl: it.itemId ? `${SITE}/auctions/item.asp?id=${it.itemId}` : p.url,179 externalId: it.itemId ?? `${s.code}:${it.title}:${it.price}`,180 rawTitle: `${it.title}${it.gradeLine ? ` — ${it.gradeLine}` : ''}`,181 imageUrls: it.image ? [it.image] : [],182 attributes,183 grade: { grader: g.grader, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: g.qualifier, certificationNumber: null },184 condition: { condition: null, conditionRaw: g.grader === 'raw' ? `${g.label ?? ''} ${g.grade ?? ''}`.trim() || null : null, completeness: null },185 observedAt: raw.fetchedAt,186 confidence: 0.7,187 parserVersion: PARSER_VERSION,188 saleType: 'auction',189 saleDate,190 price: it.price,191 currency: 'USD',192 buyerPremiumIncluded: null,193 quantity: 1,194 isBundle: t.isLot,195 location: 'US',196 auctionHouse: 'ComicLink',197 lotNumber: null,198 }),199 );200 }201 return out;202 }203}204205export default (meta: ConnectorMeta) => new ComicLinkConnector(meta);206