import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; import { dateWords, lotAttributes, makeSale, money } from '../../firecrawl/_carlib/index.js'; import { hintFromLabel, isBundleTitle, safeYear, slugFromTitle } from '../_auction-lib/categories.js'; /** * Potter & Potter Auctions (Chicago) — magic, playing cards, gambling, posters, books, photography, * pop culture. Public Bidsquare-hosted pages, plain HTTPS. See meta.json accessNotes. */ const BASE = 'https://auction.potterauctions.com'; const PARSER_VERSION = '1.0.0'; export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), url: z.string(), startText: z.string().nullable() }); export type Auction = z.infer; export const LotSchema = z.object({ lotNumber: z.string(), title: z.string(), url: z.string(), statusLabel: z.string().nullable(), priceText: z.string().nullable(), bids: z.number().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), image: z.string().nullable(), }); export type Lot = z.infer; export const PayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), totalLots: z.number().nullable(), lots: z.array(LotSchema) }); export type Payload = z.infer; function decode(s: string): string { return s .replace(/&/g, '&') .replace(/�?39;|'/g, "'") .replace(/"/g, '"') .replace(/</g, '<') .replace(/>/g, '>') .replace(/ /g, ' ') .replace(/<[^>]+>/g, ' ') .replace(/\s+/g, ' ') .trim(); } /** Past-auction list page → auctions (newest first as published). */ export function parsePastList(html: string): Auction[] { const out: Auction[] = []; const re = /data-event_id='(\d+)'\s+data-event_status='past'\s+data-event_name='([^']*)'[\s\S]*?href="(https:\/\/auction\.potterauctions\.com\/auctions\/potter-potter\/([a-z0-9-]+)-\1)"[\s\S]*?Start:\s*([^<]+) a.id === m![1])) continue; out.push({ id: m[1]!, slug: m[4]!, title: decode(m[2]!), url: m[3]!, startText: m[5]!.trim() || null }); } return out; } /** Catalog page → lots. Lot cards are server-rendered; image precedes the lot number block. */ export function parseCatalogPage(html: string, auction: Auction, page: number): Payload { const lots: Lot[] = []; const totalMatch = html.match(/(\d+)\s+Lots?\b/i) ?? html.match(/total[^0-9]{0,40}(\d+)/i); const totalLots = totalMatch ? Number(totalMatch[1]) : null; const idx: number[] = []; const marker = /
Lot\s+/g; let mm: RegExpExecArray | null; while ((mm = marker.exec(html))) idx.push(mm.index); for (let k = 0; k < idx.length; k++) { const start = idx[k]!; const end = idx[k + 1] ?? html.length; const block = html.slice(start, end); const before = html.slice(Math.max(0, start - 2500), start); const num = block.match(/^
Lot\s+([^<]+)\s*([\s\S]*?)<\/a>/); if (!num || !t) continue; const label = block.match(/id="lbl_\d+_\d+">([^<]*)]*>\s*([^<]+)]*>\s*(\d+)\s*Bids?/)?.[1]; const est = block.match(/data-exchange='\{"low_est":"([\d.]+)","high_est":"([\d.]+)"/); const img = [...before.matchAll(/ { const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2); const pagesPerAuction = Number(this.meta.config.pagesPerAuction ?? 12); const backfill = ctx.options.mode === 'backfill'; const done = new Set(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []); const listPage = backfill ? Number(ctx.options.cursor?.listPage ?? 1) : 1; const listUrl = `${BASE}/auctions/past?page=${listPage}`; await this.throttle(); const list = await ctx.fetch(listUrl, { responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parsePastList(r.html)[0]?.title ?? null, date: parsePastList(r.html)[0]?.startText ?? null } : null) }); if (!list.success || !list.html) { ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`); return; } const auctions = parsePastList(list.html).filter((a) => !done.has(a.id)); let count = 0; let processed = 0; for (const auction of auctions) { if (processed >= auctionsPerRun || ctx.signal?.aborted) break; for (let page = 1; page <= pagesPerAuction; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${auction.url}/catalog?page=${page}`; await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'status'], parse: (r) => { const f = r.html ? parseCatalogPage(r.html, auction, page).lots.find((l) => l.priceText) : null; return f ? { title: f.title, price: money(f.priceText, 'USD')?.amount ?? null, status: f.statusLabel } : null; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseCatalogPage(res.html, auction, page); if (payload.lots.length === 0) break; count++; yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (payload.totalLots !== null && page * payload.lots.length >= payload.totalLots) break; } processed++; done.add(auction.id); await ctx.setCursor({ doneAuctions: [...done].slice(-300), listPage: backfill && auctions.every((a) => done.has(a.id)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const saleDate = dateWords(p.auction.startText); if (!saleDate) return []; const out: NormalizedSale[] = []; for (const lot of p.lots) { if (!lot.priceText || !/sold/i.test(lot.statusLabel ?? '')) continue; const m = money(lot.priceText, 'USD'); if (!m) continue; const g = parseGradeFromTitle(lot.title); const categorySlug = potterCategory(p.auction.title, lot.title); const attributes = lotAttributes({ categorySlug, name: lot.title, year: safeYear(lot.title), identifiers: { potter_lot: `${p.auction.id}-${lot.lotNumber}`, bidsquare_item: lot.url.match(/-(\d+)$/)?.[1] ?? '' }, metadata: { auction_id: p.auction.id, department: p.auction.title, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, bids: lot.bids, price_is_hammer: true }, }); if (!attributes.identifiers.bidsquare_item) delete attributes.identifiers.bidsquare_item; out.push( makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber}`, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: false, auctionHouse: 'Potter & Potter Auctions', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, isBundle: isBundleTitle(lot.title), location: 'US', }), ); } return out; } } export default (meta: ConnectorMeta) => new PotterAuctionsConnector(meta);