TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { amount, certFromTitle, isBundleTitle, lotAttributes, makeSale, monthYearDate, safeYear, saleGrade, sportsCategory } from '../_g7-auctions-na-lib/index.js';56const BASE = 'https://www.cleansweepauctions.com';7const HOUSE = 'Clean Sweep Auctions';8const PARSER_VERSION = '1.0.0';9const BUYERS_PREMIUM_PCT = 22;1011export const LotPayloadSchema = z.object({12 kind: z.literal('cs_lot'),13 month: z.string(),14 monthLabel: z.string().nullable(),15 listUrl: z.string(),16 url: z.string(),17 slug: z.string(),18 title: z.string(),19 winningBidText: z.string().nullable(),20 price: z.number().nullable(),21 image: z.string().nullable(),22 description: z.string().nullable(),23});24export type LotPayload = z.infer<typeof LotPayloadSchema>;2526const MONTHS = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december'];2728/** "april-2025" → sortable key 202504; null when not a month slug. */29export function monthKey(slug: string): number | null {30 const m = slug.match(/^([a-z]+)-(\d{4})$/i);31 if (!m) return null;32 const mo = MONTHS.indexOf(m[1]!.toLowerCase());33 return mo < 0 ? null : Number(m[2]) * 100 + mo + 1;34}3536/** /past-auctions index → month slugs, newest first. */37export function parsePastIndex(htmlText: string): string[] {38 const $ = H.load(htmlText);39 const set = new Set<string>();40 $('a[href^="/past-auctions/"]').each((_, a) => {41 const slug = ($(a).attr('href') ?? '').replace(/^\/past-auctions\//, '').split(/[?#/]/)[0] ?? '';42 if (monthKey(slug) !== null) set.add(slug.toLowerCase());43 });44 return [...set].sort((a, b) => monthKey(b)! - monthKey(a)!);45}4647export interface MonthPage {48 month: string;49 monthLabel: string | null;50 page: number;51 totalPages: number | null;52 lots: Array<{ slug: string; url: string; title: string; image: string | null }>;53}5455/** /past-auctions/<month-year>?page=N → lot links (50 per page) + page count. */56export function parseMonthPage(htmlText: string, month: string, page: number): MonthPage {57 const $ = H.load(htmlText);58 const lots: MonthPage['lots'] = [];59 const seen = new Set<string>();60 const prefix = `/past-auctions/${month}/`;61 $('.auction-list-item').each((_, el) => {62 const it = $(el);63 const a = it.find(`a[href^="${prefix}"]`).first();64 const href = a.attr('href');65 const title = H.text(a);66 if (!href || !title) return;67 const slug = href.slice(prefix.length).split(/[?#]/)[0]!;68 if (!slug || seen.has(slug)) return;69 seen.add(slug);70 const img = it.find('img').attr('src') ?? null;71 lots.push({ slug, url: `${BASE}${prefix}${slug}`, title, image: img ? img.replace(/^http:/, 'https:') : null });72 });73 const pages = $('.pagination a[href*="page="]')74 .map((_, a) => Number(($(a).attr('href') ?? '').match(/page=(\d+)/)?.[1] ?? 0))75 .get()76 .filter((n) => n > 0);77 return { month, monthLabel: H.text($('h1').first()), page, totalPages: pages.length ? Math.max(...pages) : null, lots };78}7980/** Lot page → title, "Winning bid: $X", image. */81export function parseLotPage(htmlText: string, url: string, month: string, slug: string, listUrl: string): LotPayload | null {82 const $ = H.load(htmlText);83 const title = H.text($('h1').first()) ?? H.text($('h2').first());84 if (!title) return null;85 const text = $('body').text().replace(/\s+/g, ' ');86 const bid = text.match(/Winning bid:\s*(\$[\d,]+(?:\.\d{1,2})?)/i)?.[1] ?? null;87 const image = $('.card img').first().attr('src') ?? null;88 const monthLabel = H.text($('ul.crumb a[href$="' + month + '"]').first()) ?? null;89 const desc = H.text($('.col-md-8 > div p').first());90 return { kind: 'cs_lot', month, monthLabel, listUrl, url, slug, title, winningBidText: bid, price: amount(bid), image: image ? image.replace(/^http:/, 'https:') : null, description: desc };91}9293const CARD_BRANDS = /\b(topps|bowman|fleer|donruss|panini|upper deck|leaf|goudey|exhibit|play ball|t206|t205|e\d{2,3}|w\d{3}|score|pro set|skybox|hoops|o-pee-chee|opc|parkhurst|red man|kellogg'?s|hostess|post cereal|bazooka|nu-card|philadelphia|sportscaster|tcma|sgc|psa|bgs|beckett)\b/i;94const CARD_HINTS = /\b(rc|rookie|refractor|parallel|auto|autograph card|wax|unopened|pack|set break|cello|rack pack|checklist|#\d+|\d{4} .* \d{1,3}\b)\b/i;95const MEMORABILIA_WORDS = /\b(program|ticket|stub|pennant|photo|photograph|ball|bat|jersey|glove|helmet|button|pin|magazine|yearbook|guide|book|press|schedule|poster|scorecard|contract|check|letter|menu|plate|bobble|figure|puck|stick|medal|ring|patch|cap|hat|shirt|uniform|display|sign|dinner)\b/i;9697/**98 * Clean Sweep titles are terse ("1971 Topps 556 Jim McClothlin 8", "2018 Topps Update 285 Ohtani RC Ex"):99 * a year + card brand + number without memorabilia words is a card even when the shared mapper cannot tell.100 */101export function cleanSweepCategory(title: string): string {102 const generic = sportsCategory(title);103 if (generic.endsWith('_cards') || generic === 'pokemon' || generic === 'magic_the_gathering') return generic;104 const cardLike = CARD_BRANDS.test(title) && (CARD_HINTS.test(title) || /\b(19|20)\d{2}\b/.test(title)) && !MEMORABILIA_WORDS.test(title);105 if (!cardLike) return generic;106 const sport = /\b(basketball|nba|jordan|lebron|kobe|hoops|skybox)\b/i.test(title) ? 'basketball_cards' : /\b(football|nfl|pro set|brady|mahomes|montana|payton)\b/i.test(title) ? 'football_cards' : /\b(hockey|nhl|o-pee-chee|opc|parkhurst|gretzky|orr|howe)\b/i.test(title) ? 'hockey_cards' : /\b(boxing|golf|tennis|wrestling|nascar|racing|olympic|soccer)\b/i.test(title) ? 'other_sports_cards' : 'baseball_cards';107 return sport;108}109110/** Bundles: "(3 pcs)", "Lot of 12", "(27 different)". */111export function isBundle(title: string): boolean {112 return isBundleTitle(title) || /\(\s*\d+\s*(?:pcs?|pieces|different|cards|items)\s*\)/i.test(title) || /\b\d+\s*(?:pcs?|different)\b/i.test(title);113}114115interface Cursor {116 doneMonths: string[];117 current: { month: string; page: number; index: number } | null;118 done?: boolean;119}120121/**122 * Clean Sweep Auctions — past-auction archive (2008 → present). Public pages only:123 * /past-auctions (one entry per closed monthly auction) → /past-auctions/<month-year>?page=N (50 lots)124 * → /past-auctions/<month-year>/<slug> ("Winning bid: $X", hammer before the 22 % buyer's premium).125 * The source only dates lots by the auction month, so saleDate is the first of that month (precision flagged).126 */127export class CleanSweepConnector extends BaseConnector {128 readonly version = '1.0.0';129 readonly parserVersion = PARSER_VERSION;130 protected override minIntervalMs = 2000;131132 private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> {133 await this.throttle(url);134 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 45_000 });135 if (!res.success || !res.html) {136 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);137 return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt };138 }139 return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt };140 }141142 private readCursor(ctx: CrawlContext): Cursor {143 const c = ctx.options.cursor ?? {};144 const cur = c.current as Cursor['current'] | undefined;145 return { doneMonths: Array.isArray(c.doneMonths) ? (c.doneMonths as string[]) : [], current: cur && typeof cur === 'object' && cur.month ? { month: cur.month, page: Number(cur.page ?? 1), index: Number(cur.index ?? 0) } : null };146 }147148 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {149 const mode = ctx.options.mode;150 const cfg = this.meta.config;151 const lotsPerRun = mode === 'probe' ? (ctx.options.limit ?? 3) : Number(cfg.lotsPerRun ?? 150);152 const pagesPerRun = mode === 'probe' ? 1 : Number(cfg.pagesPerRun ?? 4);153 const cursor = this.readCursor(ctx);154 const done = new Set(cursor.doneMonths);155156 // Seeds: month slugs / month URLs / lot URLs.157 const seedLots: Array<{ month: string; slug: string }> = [];158 const seedMonths: string[] = [];159 for (const s of ctx.options.seeds ?? []) {160 const m = s.match(/past-auctions\/([a-z]+-\d{4})(?:\/([^/?#]+))?/i) ?? s.match(/^([a-z]+-\d{4})$/i);161 if (!m) continue;162 if (m[2]) seedLots.push({ month: m[1]!.toLowerCase(), slug: m[2] });163 else seedMonths.push(m[1]!.toLowerCase());164 }165 let count = 0;166 let items = 0;167 for (const sl of seedLots) {168 if (ctx.signal?.aborted || this.reached(ctx, count)) return;169 const listUrl = `${BASE}/past-auctions/${sl.month}`;170 const url = `${listUrl}/${sl.slug}`;171 const r = await this.html(ctx, url);172 const payload = r.html ? parseLotPage(r.html, url, sl.month, sl.slug, listUrl) : null;173 if (!payload) continue;174 count++;175 yield { url, externalId: `${sl.month}/${sl.slug}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt };176 }177 if (seedLots.length && !seedMonths.length) return;178179 let months = seedMonths;180 if (!months.length) {181 const idx = await this.html(ctx, `${BASE}/past-auctions`);182 if (!idx.html) return;183 months = parsePastIndex(idx.html);184 if (!months.length) {185 ctx.anomaly('selector_missing', '/past-auctions: no month links found');186 return;187 }188 }189 // Backfill and incremental both walk newest → oldest; incremental stops after the first (newest) pending month.190 const pending = months.filter((m) => !done.has(m));191 const queue = cursor.current && pending.includes(cursor.current.month) ? [cursor.current.month, ...pending.filter((m) => m !== cursor.current!.month)] : pending;192 if (!queue.length) {193 if (mode === 'backfill') await ctx.setCursor({ doneMonths: [...done], current: null, done: true, updatedAt: new Date().toISOString() });194 return;195 }196 const monthsThisRun = mode === 'incremental' ? queue.slice(0, 1) : queue;197 let pagesFetched = 0;198 let reachedDate: Date | null = null;199 for (const month of monthsThisRun) {200 if (ctx.signal?.aborted || this.reached(ctx, count) || items >= lotsPerRun || pagesFetched >= pagesPerRun) break;201 const listUrl = `${BASE}/past-auctions/${month}`;202 let page = cursor.current?.month === month ? cursor.current.page : 1;203 let index = cursor.current?.month === month ? cursor.current.index : 0;204 let monthComplete = false;205 while (!ctx.signal?.aborted && pagesFetched < pagesPerRun) {206 const r = await this.html(ctx, page > 1 ? `${listUrl}?page=${page}` : listUrl);207 pagesFetched++;208 if (!r.html) break;209 const mp = parseMonthPage(r.html, month, page);210 if (mp.lots.length === 0 && page === 1) ctx.anomaly('selector_missing', `${listUrl}: no lots found`);211 let stop = false;212 for (let i = index; i < mp.lots.length; i++) {213 if (ctx.signal?.aborted || this.reached(ctx, count) || items >= lotsPerRun) {214 stop = true;215 break;216 }217 const lot = mp.lots[i]!;218 const r2 = await this.html(ctx, lot.url);219 index = i + 1;220 await ctx.setCursor({ doneMonths: [...done], current: { month, page, index }, updatedAt: new Date().toISOString() });221 if (!r2.html) continue;222 const payload = parseLotPage(r2.html, lot.url, month, lot.slug, listUrl);223 if (!payload) {224 ctx.anomaly('parse_failure_page', lot.url);225 continue;226 }227 if (!payload.image && lot.image) payload.image = lot.image;228 if (payload.price === null) ctx.anomaly('price_parse_failure', `${lot.url}: ${payload.winningBidText ?? 'no winning bid'}`);229 count++;230 items++;231 yield { url: lot.url, externalId: `${month}/${lot.slug}`, kind: 'sale', engine: 'api', httpStatus: r2.status, payload, fetchedAt: r2.fetchedAt };232 }233 if (stop) break;234 const last = mp.totalPages !== null ? page >= mp.totalPages : mp.lots.length < 50;235 if (last) {236 monthComplete = true;237 break;238 }239 page++;240 index = 0;241 await ctx.setCursor({ doneMonths: [...done], current: { month, page, index }, updatedAt: new Date().toISOString() });242 }243 if (monthComplete) {244 done.add(month);245 reachedDate = monthYearDate(month);246 await ctx.setCursor({ doneMonths: [...done], current: null, updatedAt: new Date().toISOString() });247 }248 await ctx.progress({ page: months.filter((m) => done.has(m)).length, totalPages: months.length, itemsProcessed: items, reachedDate, cursor: { doneMonths: [...done], current: monthComplete ? null : { month, page, index } } });249 }250 if (mode === 'backfill' && months.every((m) => done.has(m))) await ctx.setCursor({ doneMonths: [...done], current: null, done: true, updatedAt: new Date().toISOString() });251 }252253 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {254 const p = LotPayloadSchema.parse(raw.payload);255 if (p.price === null || p.price <= 0) return [];256 const saleDate = monthYearDate(p.month);257 if (!saleDate) return [];258 const g = saleGrade(p.title);259 const graded = Boolean(g.grader && g.grade);260 const slug = /\b(signed|autograph)/i.test(p.title) && !/\b(baseball|football|basketball|hockey|boxing|golf|tennis|nascar|wrestling|topps|bowman|fleer|card|ball|bat|jersey|program|ticket|photo)\b/i.test(p.title) ? 'autographs' : cleanSweepCategory(p.title);261 const attributes = lotAttributes({ categorySlug: slug, name: p.title, year: safeYear(p.title), identifiers: { clean_sweep_lot_slug: `${p.month}/${p.slug}` }, metadata: { auction_month: p.month, auction_label: p.monthLabel, sale_date_precision: 'month', buyers_premium_pct: BUYERS_PREMIUM_PCT, hammer_price: p.price } });262 const sale = makeSale({ meta: this.meta, sourceUrl: p.url, externalId: `${p.month}/${p.slug}`, rawTitle: p.title, description: p.description, attributes, price: p.price, currency: 'USD', saleDate, buyerPremiumIncluded: false, auctionHouse: HOUSE, imageUrls: p.image ? [p.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundle(p.title), confidence: graded ? 0.75 : 0.7 });263 sale.grade.qualifier = g.qualifier;264 sale.grade.certificationNumber = certFromTitle(p.title);265 return [sale];266 }267}268269export default (meta: ConnectorMeta) => new CleanSweepConnector(meta);270