TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';5import { bidsquareDate, gradeOf, lotAttributes, makeLot, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory, toyGrade, type BidsquareCatalog } from '../_memorabilia-lib/index.js';6import { makeSale } from '../../firecrawl/_carlib/index.js';78const PARSER_VERSION = '1.0.0';910export const CatalogPayloadSchema = z.object({11 kind: z.literal('catalog_page'),12 house: z.string(),13 event: z.object({ id: z.string(), name: z.string(), url: z.string(), status: z.enum(['upcoming', 'live', 'past', 'unknown']), startDate: z.string().nullable(), endDate: z.string().nullable() }),14 page: z.number(),15 totalPages: z.number().nullable(),16 items: z.array(17 z.object({18 itemId: z.string(),19 url: z.string(),20 title: z.string(),21 lotNumber: z.string().nullable(),22 image: z.string().nullable(),23 priceLabel: z.string().nullable(),24 price: z.number().nullable(),25 bids: z.number().nullable(),26 estimateLow: z.number().nullable(),27 estimateHigh: z.number().nullable(),28 startsAt: z.number().nullable(),29 endsAt: z.number().nullable(),30 }),31 ),32});33export type CatalogPayload = z.infer<typeof CatalogPayloadSchema>;3435export interface BidsquareHouseOptions {36 base: string;37 house: string;38 /** map a lot title to a taxonomy slug */39 category: (title: string) => string | null;40 /** buyer's premium included in the displayed "Sold for" price? null when the house does not state it */41 buyerPremiumIncluded: boolean | null;42 idKey: string;43 location: string | null;44}4546/**47 * Bidsquare-hosted auction catalogs (SCP Auctions, Hake's). Public pages only:48 * /auctions (upcoming), /auctions/past?page=N, /auctions/<house>/<slug>-<id>/catalog?page=N49 * Past catalogs show "Sold for $X" per lot → sales dated by the event end (schema.org Event JSON-LD);50 * upcoming catalogs show current/starting bid + estimate → auction lots for the calendar.51 */52export class BidsquareHouseConnector extends BaseConnector {53 readonly version = '1.0.0';54 readonly parserVersion = PARSER_VERSION;55 protected override minIntervalMs = 2000;5657 constructor(meta: ConnectorMeta, protected readonly opts: BidsquareHouseOptions) {58 super(meta);59 }6061 protected async fetchHtml(ctx: CrawlContext, url: string): Promise<{ html: string | null; engine: 'api' | 'feed' | 'firecrawl' | 'scrapfly' | 'browser' | 'manual'; status: number | null; fetchedAt: Date }> {62 await this.throttle();63 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 45_000 });64 if (!res.success || !res.html) {65 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);66 return { html: null, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };67 }68 return { html: res.html, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };69 }7071 /** Events to visit this run: past (sold) first, then upcoming (lots). Subclasses may override. */72 protected async listEvents(ctx: CrawlContext): Promise<Array<{ id: string; name: string; url: string; status: string }>> {73 const backfill = ctx.options.mode === 'backfill';74 const page = backfill ? Number(ctx.options.cursor?.pastPage ?? 1) : 1;75 const out: Array<{ id: string; name: string; url: string; status: string }> = [];76 const past = await this.fetchHtml(ctx, `${this.opts.base}/auctions/past${page > 1 ? `?page=${page}` : ''}`);77 if (past.html) out.push(...parseBidsquareEvents(past.html).map((e) => ({ ...e, status: 'past' })));78 if (this.meta.config.includeUpcoming !== false) {79 const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`);80 if (up.html) out.push(...parseBidsquareEvents(up.html).filter((e) => e.status !== 'past').map((e) => ({ ...e, status: e.status === 'past' ? 'past' : 'upcoming' })));81 }82 return out;83 }8485 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {86 const eventsPerRun = Number(this.meta.config.eventsPerRun ?? 2);87 const maxPages = Number(this.meta.config.pagesPerEvent ?? 8);88 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneEvents) ? (ctx.options.cursor!.doneEvents as string[]) : []);89 const events = await this.listEvents(ctx);90 let processed = 0;91 let count = 0;92 for (const ev of events) {93 if (ctx.signal?.aborted || processed >= eventsPerRun) break;94 if (ev.status === 'past' && done.has(ev.id)) continue;95 const catalogBase = `${ev.url.replace(/\/$/, '')}/catalog`;96 let total: number | null = null;97 for (let page = 1; page <= maxPages; page++) {98 if (ctx.signal?.aborted || this.reached(ctx, count)) break;99 const url = page === 1 ? catalogBase : `${catalogBase}?page=${page}`;100 const r = await this.fetchHtml(ctx, url);101 if (!r.html) break;102 const cat = parseBidsquareCatalog(r.html, page);103 if (!cat || cat.items.length === 0) break;104 total = cat.totalPages ?? total;105 const payload: CatalogPayload = { kind: 'catalog_page', house: this.opts.house, event: { ...cat.event, name: cat.event.name || ev.name, url: cat.event.url || ev.url }, page, totalPages: total, items: cat.items };106 count++;107 yield { url, externalId: `event:${cat.event.id}:page:${page}`, kind: cat.event.status === 'past' ? 'sale' : 'auction_lot', engine: r.engine, httpStatus: r.status, payload, fetchedAt: r.fetchedAt };108 if (total !== null && page >= total) break;109 }110 processed++;111 if (ev.status === 'past') done.add(ev.id);112 }113 const pastPage = Number(ctx.options.cursor?.pastPage ?? 1);114 await ctx.setCursor({ doneEvents: [...done].slice(-500), pastPage: ctx.options.mode === 'backfill' && events.filter((e) => e.status === 'past').every((e) => done.has(e.id)) ? pastPage + 1 : pastPage, updatedAt: new Date().toISOString() });115 }116117 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {118 const p = CatalogPayloadSchema.parse(raw.payload);119 const out: NormalizedRecord[] = [];120 const endDate = bidsquareDate(p.event.endDate);121 const startDate = bidsquareDate(p.event.startDate);122 for (const it of p.items) {123 const slug = this.opts.category(it.title);124 if (!slug) continue;125 const g = gradeOf(it.title);126 const tg = toyGrade(it.title);127 const attributes = lotAttributes({128 categorySlug: slug,129 name: it.title,130 year: safeYear(it.title),131 identifiers: { [this.opts.idKey]: it.itemId },132 metadata: { event_id: p.event.id, event_name: p.event.name, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bids: it.bids, ...(tg ? { toy_grade: `${tg.company} ${tg.grade}` } : {}) },133 });134 const common = { meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade };135 const sold = p.event.status === 'past' && /sold/i.test(it.priceLabel ?? '') && it.price !== null && it.price > 0;136 if (sold) {137 const saleDate = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;138 if (!saleDate) continue;139 out.push(makeSale({ ...common, price: it.price!, currency: 'USD', saleDate, buyerPremiumIncluded: this.opts.buyerPremiumIncluded, auctionHouse: this.opts.house, lotNumber: it.lotNumber, location: this.opts.location, isBundle: isBundleTitle(it.title) }));140 } else if (p.event.status !== 'past') {141 const endsAt = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;142 const status = startDate && startDate.getTime() > Date.now() ? 'upcoming' : 'live';143 out.push(makeLot({ ...common, auctionHouse: this.opts.house, auctionName: p.event.name, lotNumber: it.lotNumber, startsAt: startDate, endsAt, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: /bid/i.test(it.priceLabel ?? '') ? it.price : null, currency: 'USD', status, location: this.opts.location, confidence: 0.85 }));144 }145 }146 return out;147 }148}149150export function scpCategory(title: string): string {151 return sportsCategory(title);152}153154export default (meta: ConnectorMeta) =>155 new BidsquareHouseConnector(meta, {156 base: 'https://catalogs.scpauctions.com',157 house: 'SCP Auctions',158 category: scpCategory,159 // SCP does not state on the catalog page whether "Sold for" includes the buyer's premium.160 buyerPremiumIncluded: null,161 idKey: 'scp_item_id',162 location: 'US',163 });164165export type { BidsquareCatalog };166