… … TITLE
SOLD in CGC 9.4 NM
$1,365. */ export function parseResultsPage(htmlText: string): ResultItem[] { const $ = H.load(htmlText); const out: ResultItem[] = []; $('div.bookcover').each((_, el) => { const $el = $(el); const href = $el.find('a[href*="item.asp"]').first().attr('href') ?? ''; const itemId = href.match(/id=(\d+)/)?.[1] ?? null; const imgSrc = $el.find('img').first().attr('src') ?? null; const b = $el.find('b').last(); const parts = (b.html() ?? '') .split(//i) .map((s) => H.load(`${s}`)('x').text().replace(/\s+/g, ' ').trim()) .filter(Boolean); if (parts.length < 2) return; const priceText = parts.find((p) => /\$\s?[\d,]+/.test(p)) ?? ''; const price = usd(priceText); const title = parts[0]!; const gradeLine = parts.find((p) => /^SOLD\b/i.test(p))?.replace(/^SOLD\s+(?:in|as)?\s*/i, '') ?? null; if (!title || !price) return; out.push({ itemId, title, gradeLine, priceText, price, image: imgSrc ? H.absUrl(SITE, imgSrc.replace(/^\/\.\//, '/')) : null }); }); return out; } export class ComicLinkConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; private async fetchHtml(ctx: CrawlContext, url: string, expectItems: boolean): Promise { await this.throttle(url); const res = await ctx.fetch(url, { engines: ['scrapfly'], renderJs: false, country: 'us', timeoutMs: 90_000, expect: expectItems ? ['title', 'price', 'status'] : ['title'], parse: (r) => { if (!r.html) return null; if (!expectItems) return { title: /generate_highlights/.test(r.html) ? 'ok' : null }; const first = parseResultsPage(r.html)[0]; return first ? { title: first.title, price: first.price, status: 'sold' } : null; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return null; } return res.html; } async *crawl(ctx: CrawlContext): AsyncIterable { const includeArt = Boolean(this.meta.config.includeArtSessions ?? false); const backfill = ctx.options.mode === 'backfill'; const perRun = ctx.options.mode === 'probe' ? 1 : Number(backfill ? this.meta.config.backfillSessionsPerRun ?? 30 : this.meta.config.sessionsPerRun ?? 2); const scheduleHtml = await this.fetchHtml(ctx, SCHEDULE, false); if (!scheduleHtml) return; let sessions = parseSchedule(scheduleHtml).filter((s) => includeArt || !s.isArt); if (ctx.options.seeds?.length) sessions = sessions.filter((s) => ctx.options.seeds!.includes(s.code)); if (!sessions.length) { ctx.anomaly('selector_missing', 'no result sessions found on the auction schedule page'); return; } const cursor = (ctx.options.cursor ?? {}) as { index?: number; done?: boolean }; let index = backfill ? Math.max(0, Number(cursor.index ?? 0)) : 0; let count = 0; for (let n = 0; n < perRun && index < sessions.length; n++, index++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const s = sessions[index]!; const url = `${SITE}/results/generate_highlights.asp?output=${encodeURIComponent(s.code)}`; const page = await this.fetchHtml(ctx, url, true); if (!page) continue; const items = parseResultsPage(page); if (!items.length) { ctx.anomaly('parse_failure_page', url); continue; } if (!s.endMonth) ctx.anomaly('date_parse_failure', `${s.code}: no end month in label "${s.label}"`); const payload: PagePayload = { kind: 'results_page', url, session: s, items }; count++; yield { url, externalId: `results:${s.code}`, kind: 'sale', engine: 'scrapfly', httpStatus: 200, payload, snapshot: page, fetchedAt: new Date() }; if (backfill) { await ctx.setCursor({ index: index + 1, code: s.code, updatedAt: new Date().toISOString() }); await ctx.progress({ page: index + 1, totalPages: sessions.length, itemsProcessed: count, reachedDate: s.endYear && s.endMonth ? new Date(Date.UTC(s.endYear, s.endMonth - 1, 1)) : null }); } } if (backfill && index >= sessions.length) await ctx.setCursor({ done: true, index: 0, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const s = p.session; if (!s.endYear || !s.endMonth) return []; // no source date → nothing we can honestly call a sale const saleDate = new Date(Date.UTC(s.endYear, s.endMonth - 1, 1)); const out: NormalizedRecord[] = []; for (const it of p.items) { const t = parseComicTitle(it.title); const g = parseComicGrade(it.gradeLine ?? it.title); const artItem = s.isArt || /\b(original art|cover art|splash|page \d+|painting|sketch|illustration|commission)\b/i.test(it.title); const categorySlug = artItem ? 'art' : (slugFromTitle(it.title, 'comics') ?? 'independent_comics'); const attributes = attrs({ categorySlug, series: artItem ? null : t.series, set: artItem ? null : t.series, name: artItem ? it.title : `${t.series}${t.issue ? ` #${t.issue}` : ''}`, number: artItem ? null : t.issue, year: t.year, variant: t.variant, language: 'English', country: 'US', identifiers: it.itemId ? { comiclink_item_id: it.itemId } : {}, metadata: { session_code: s.code, session_label: s.label, sale_date_precision: 'month', grade_line: it.gradeLine, grade_label: g.label, comic_art: artItem, highlights_page: true }, }); out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.itemId ? `${SITE}/auctions/item.asp?id=${it.itemId}` : p.url, externalId: it.itemId ?? `${s.code}:${it.title}:${it.price}`, rawTitle: `${it.title}${it.gradeLine ? ` — ${it.gradeLine}` : ''}`, imageUrls: it.image ? [it.image] : [], attributes, grade: { grader: g.grader, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: g.qualifier, certificationNumber: null }, condition: { condition: null, conditionRaw: g.grader === 'raw' ? `${g.label ?? ''} ${g.grade ?? ''}`.trim() || null : null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.7, parserVersion: PARSER_VERSION, saleType: 'auction', saleDate, price: it.price, currency: 'USD', buyerPremiumIncluded: null, quantity: 1, isBundle: t.isLot, location: 'US', auctionHouse: 'ComicLink', lotNumber: null, }), ); } return out; } } export default (meta: ConnectorMeta) => new ComicLinkConnector(meta);