TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';5import { dateWords, lotAttributes, makeSale, money } from '../../firecrawl/_carlib/index.js';6import { hintFromLabel, isBundleTitle, safeYear, slugFromTitle } from '../_auction-lib/categories.js';78/**9 * Potter & Potter Auctions (Chicago) — magic, playing cards, gambling, posters, books, photography,10 * pop culture. Public Bidsquare-hosted pages, plain HTTPS. See meta.json accessNotes.11 */1213const BASE = 'https://auction.potterauctions.com';14const PARSER_VERSION = '1.0.0';1516export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), url: z.string(), startText: z.string().nullable() });17export type Auction = z.infer<typeof AuctionSchema>;1819export const LotSchema = z.object({20 lotNumber: z.string(),21 title: z.string(),22 url: z.string(),23 statusLabel: z.string().nullable(),24 priceText: z.string().nullable(),25 bids: z.number().nullable(),26 estimateLow: z.number().nullable(),27 estimateHigh: z.number().nullable(),28 image: z.string().nullable(),29});30export type Lot = z.infer<typeof LotSchema>;3132export const PayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), totalLots: z.number().nullable(), lots: z.array(LotSchema) });33export type Payload = z.infer<typeof PayloadSchema>;3435function decode(s: string): string {36 return s37 .replace(/&/g, '&')38 .replace(/�?39;|'/g, "'")39 .replace(/"/g, '"')40 .replace(/</g, '<')41 .replace(/>/g, '>')42 .replace(/ /g, ' ')43 .replace(/<[^>]+>/g, ' ')44 .replace(/\s+/g, ' ')45 .trim();46}4748/** Past-auction list page → auctions (newest first as published). */49export function parsePastList(html: string): Auction[] {50 const out: Auction[] = [];51 const re = /data-event_id='(\d+)'\s+data-event_status='past'\s+data-event_name='([^']*)'[\s\S]*?href="(https:\/\/auction\.potterauctions\.com\/auctions\/potter-potter\/([a-z0-9-]+)-\1)"[\s\S]*?Start:\s*([^<]+)</g;52 let m: RegExpExecArray | null;53 while ((m = re.exec(html))) {54 if (out.some((a) => a.id === m![1])) continue;55 out.push({ id: m[1]!, slug: m[4]!, title: decode(m[2]!), url: m[3]!, startText: m[5]!.trim() || null });56 }57 return out;58}5960/** Catalog page → lots. Lot cards are server-rendered; image precedes the lot number block. */61export function parseCatalogPage(html: string, auction: Auction, page: number): Payload {62 const lots: Lot[] = [];63 const totalMatch = html.match(/(\d+)\s+Lots?\b/i) ?? html.match(/total[^0-9]{0,40}(\d+)/i);64 const totalLots = totalMatch ? Number(totalMatch[1]) : null;65 const idx: number[] = [];66 const marker = /<div class="lot_Num">Lot\s+/g;67 let mm: RegExpExecArray | null;68 while ((mm = marker.exec(html))) idx.push(mm.index);69 for (let k = 0; k < idx.length; k++) {70 const start = idx[k]!;71 const end = idx[k + 1] ?? html.length;72 const block = html.slice(start, end);73 const before = html.slice(Math.max(0, start - 2500), start);74 const num = block.match(/^<div class="lot_Num">Lot\s+([^<]+)</)?.[1]?.trim();75 const t = block.match(/lot_title">\s*<a href="([^"]+)">([\s\S]*?)<\/a>/);76 if (!num || !t) continue;77 const label = block.match(/id="lbl_\d+_\d+">([^<]*)</)?.[1]?.trim() ?? null;78 const price = block.match(/id="tcb_\d+_\d+"[^>]*>\s*([^<]+)</)?.[1]?.trim() ?? null;79 const bids = block.match(/id="tbc_\d+_\d+"[^>]*>\s*(\d+)\s*Bids?/)?.[1];80 const est = block.match(/data-exchange='\{"low_est":"([\d.]+)","high_est":"([\d.]+)"/);81 const img = [...before.matchAll(/<img src="(https:\/\/s1\.img\.bidsquare\.com\/item\/[^"]+)"/g)].pop()?.[1] ?? null;82 lots.push({83 lotNumber: num,84 title: decode(t[2]!),85 url: t[1]!,86 statusLabel: label,87 priceText: price,88 bids: bids ? Number(bids) : null,89 estimateLow: est ? Number(est[1]) : null,90 estimateHigh: est ? Number(est[2]) : null,91 image: img,92 });93 }94 return { kind: 'catalog_page', auction, page, totalLots, lots };95}9697/** Potter departments → taxonomy slugs (never guesses beyond keywords; magicana falls back to antiques). */98export function potterCategory(auctionTitle: string, lotTitle: string): string {99 const t = `${auctionTitle} ${lotTitle}`.toLowerCase();100 if (/playing card|deck of cards|\bdecks?\b|transformation deck/.test(t)) return 'playing_cards';101 if (/gambling|casino|poker chip|roulette|faro|dice\b|slot machine|cheating/.test(t)) return 'casino_memorabilia';102 if (/coin-op|coin op|vending|arcade|trade stimulator|mutoscope/.test(t)) return 'vending_machines';103 if (/circus|sideshow|carnival|advertising|sign\b|broadside|handbill|trade card/.test(t) && !/book/.test(t)) return 'advertising';104 if (/hollywood|film|movie|cinema|screen-used|prop\b/.test(t)) return /poster|lobby card|one[- ]sheet/.test(t) ? 'movie_posters' : 'movie_memorabilia';105 if (/poster|lithograph poster|one[- ]sheet|window card/.test(t)) return 'advertising';106 if (/photograph|daguerreotype|tintype|ambrotype|cabinet card|cdv|carte de visite|gelatin silver|albumen/.test(t)) return 'photography';107 if (/autograph|signed letter|signed photograph|inscribed/.test(t) && /letter|photograph|card\b/.test(t)) return 'autographs';108 if (/manuscript|archive|letter\b|diary|document/.test(t) && !/book/.test(t)) return 'historical_documents';109 if (/rock|jazz|blues|punk|concert|album|vinyl|guitar/.test(t)) return 'music_memorabilia';110 if (/baseball|boxing|basketball|football|wrestling|sport/.test(t)) return 'sports_memorabilia';111 if (/\btoy\b|toys|doll|game\b|puzzle|magic set|magic kit/.test(t)) return 'vintage_toys';112 if (/book|volume|vols?\.|edition|manuscript|pamphlet|periodical|magazine/.test(t)) return 'books';113 // Conjuring apparatus ("Okito Coin Box", "Cups and Balls", "Card Box") is not coins/cards/toys: keep it under antiques.114 if (/magic|conjur|magicana|houdini|legerdemain|illusion/i.test(auctionTitle)) return 'antiques';115 const hinted = slugFromTitle(lotTitle, hintFromLabel(auctionTitle));116 if (hinted) return hinted;117 return 'antiques';118}119120export class PotterAuctionsConnector extends BaseConnector {121 readonly version = '1.0.0';122 readonly parserVersion = PARSER_VERSION;123 protected override minIntervalMs = 2000;124125 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {126 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);127 const pagesPerAuction = Number(this.meta.config.pagesPerAuction ?? 12);128 const backfill = ctx.options.mode === 'backfill';129 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);130 const listPage = backfill ? Number(ctx.options.cursor?.listPage ?? 1) : 1;131 const listUrl = `${BASE}/auctions/past?page=${listPage}`;132 await this.throttle();133 const list = await ctx.fetch(listUrl, { responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parsePastList(r.html)[0]?.title ?? null, date: parsePastList(r.html)[0]?.startText ?? null } : null) });134 if (!list.success || !list.html) {135 ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);136 return;137 }138 const auctions = parsePastList(list.html).filter((a) => !done.has(a.id));139 let count = 0;140 let processed = 0;141 for (const auction of auctions) {142 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;143 for (let page = 1; page <= pagesPerAuction; page++) {144 if (ctx.signal?.aborted || this.reached(ctx, count)) break;145 const url = `${auction.url}/catalog?page=${page}`;146 await this.throttle();147 const res = await ctx.fetch(url, {148 responseType: 'text',149 expect: ['title', 'price', 'status'],150 parse: (r) => {151 const f = r.html ? parseCatalogPage(r.html, auction, page).lots.find((l) => l.priceText) : null;152 return f ? { title: f.title, price: money(f.priceText, 'USD')?.amount ?? null, status: f.statusLabel } : null;153 },154 });155 if (!res.success || !res.html) {156 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);157 break;158 }159 const payload = parseCatalogPage(res.html, auction, page);160 if (payload.lots.length === 0) break;161 count++;162 yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };163 if (payload.totalLots !== null && page * payload.lots.length >= payload.totalLots) break;164 }165 processed++;166 done.add(auction.id);167 await ctx.setCursor({ doneAuctions: [...done].slice(-300), listPage: backfill && auctions.every((a) => done.has(a.id)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() });168 }169 }170171 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {172 const p = PayloadSchema.parse(raw.payload);173 const saleDate = dateWords(p.auction.startText);174 if (!saleDate) return [];175 const out: NormalizedSale[] = [];176 for (const lot of p.lots) {177 if (!lot.priceText || !/sold/i.test(lot.statusLabel ?? '')) continue;178 const m = money(lot.priceText, 'USD');179 if (!m) continue;180 const g = parseGradeFromTitle(lot.title);181 const categorySlug = potterCategory(p.auction.title, lot.title);182 const attributes = lotAttributes({183 categorySlug,184 name: lot.title,185 year: safeYear(lot.title),186 identifiers: { potter_lot: `${p.auction.id}-${lot.lotNumber}`, bidsquare_item: lot.url.match(/-(\d+)$/)?.[1] ?? '' },187 metadata: { auction_id: p.auction.id, department: p.auction.title, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, bids: lot.bids, price_is_hammer: true },188 });189 if (!attributes.identifiers.bidsquare_item) delete attributes.identifiers.bidsquare_item;190 out.push(191 makeSale({192 meta: this.meta,193 sourceUrl: lot.url,194 externalId: `${p.auction.id}-${lot.lotNumber}`,195 rawTitle: lot.title,196 attributes,197 price: m.amount,198 currency: 'USD',199 saleDate,200 buyerPremiumIncluded: false,201 auctionHouse: 'Potter & Potter Auctions',202 lotNumber: lot.lotNumber,203 imageUrls: lot.image ? [lot.image] : [],204 observedAt: raw.fetchedAt,205 parserVersion: PARSER_VERSION,206 grader: g.grader && g.grader !== 'raw' ? g.grader : null,207 grade: g.grader && g.grader !== 'raw' ? g.grade : null,208 isBundle: isBundleTitle(lot.title),209 location: 'US',210 }),211 );212 }213 return out;214 }215}216217export default (meta: ConnectorMeta) => new PotterAuctionsConnector(meta);218