import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { isBundleTitle, safeYear } from '../_auction-lib/categories.js'; import { bidsquareDate, gradeOf, lotAttributes, makeLot, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory, toyGrade, type BidsquareCatalog } from '../_memorabilia-lib/index.js'; import { makeSale } from '../../firecrawl/_carlib/index.js'; const PARSER_VERSION = '1.0.0'; export const CatalogPayloadSchema = z.object({ kind: z.literal('catalog_page'), house: z.string(), event: z.object({ id: z.string(), name: z.string(), url: z.string(), status: z.enum(['upcoming', 'live', 'past', 'unknown']), startDate: z.string().nullable(), endDate: z.string().nullable() }), page: z.number(), totalPages: z.number().nullable(), items: z.array( z.object({ itemId: z.string(), url: z.string(), title: z.string(), lotNumber: z.string().nullable(), image: z.string().nullable(), priceLabel: z.string().nullable(), price: z.number().nullable(), bids: z.number().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), startsAt: z.number().nullable(), endsAt: z.number().nullable(), }), ), }); export type CatalogPayload = z.infer; export interface BidsquareHouseOptions { base: string; house: string; /** map a lot title to a taxonomy slug */ category: (title: string) => string | null; /** buyer's premium included in the displayed "Sold for" price? null when the house does not state it */ buyerPremiumIncluded: boolean | null; idKey: string; location: string | null; } /** * Bidsquare-hosted auction catalogs (SCP Auctions, Hake's). Public pages only: * /auctions (upcoming), /auctions/past?page=N, /auctions//-/catalog?page=N * Past catalogs show "Sold for $X" per lot → sales dated by the event end (schema.org Event JSON-LD); * upcoming catalogs show current/starting bid + estimate → auction lots for the calendar. */ export class BidsquareHouseConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; constructor(meta: ConnectorMeta, protected readonly opts: BidsquareHouseOptions) { super(meta); } protected async fetchHtml(ctx: CrawlContext, url: string): Promise<{ html: string | null; engine: 'api' | 'feed' | 'firecrawl' | 'scrapfly' | 'browser' | 'manual'; status: number | null; fetchedAt: Date }> { await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 45_000 }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return { html: null, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt }; } return { html: res.html, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt }; } /** Events to visit this run: past (sold) first, then upcoming (lots). Subclasses may override. */ protected async listEvents(ctx: CrawlContext): Promise> { const backfill = ctx.options.mode === 'backfill'; const page = backfill ? Number(ctx.options.cursor?.pastPage ?? 1) : 1; const out: Array<{ id: string; name: string; url: string; status: string }> = []; const past = await this.fetchHtml(ctx, `${this.opts.base}/auctions/past${page > 1 ? `?page=${page}` : ''}`); if (past.html) out.push(...parseBidsquareEvents(past.html).map((e) => ({ ...e, status: 'past' }))); if (this.meta.config.includeUpcoming !== false) { const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`); if (up.html) out.push(...parseBidsquareEvents(up.html).filter((e) => e.status !== 'past').map((e) => ({ ...e, status: e.status === 'past' ? 'past' : 'upcoming' }))); } return out; } async *crawl(ctx: CrawlContext): AsyncIterable { const eventsPerRun = Number(this.meta.config.eventsPerRun ?? 2); const maxPages = Number(this.meta.config.pagesPerEvent ?? 8); const done = new Set(Array.isArray(ctx.options.cursor?.doneEvents) ? (ctx.options.cursor!.doneEvents as string[]) : []); const events = await this.listEvents(ctx); let processed = 0; let count = 0; for (const ev of events) { if (ctx.signal?.aborted || processed >= eventsPerRun) break; if (ev.status === 'past' && done.has(ev.id)) continue; const catalogBase = `${ev.url.replace(/\/$/, '')}/catalog`; let total: number | null = null; for (let page = 1; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = page === 1 ? catalogBase : `${catalogBase}?page=${page}`; const r = await this.fetchHtml(ctx, url); if (!r.html) break; const cat = parseBidsquareCatalog(r.html, page); if (!cat || cat.items.length === 0) break; total = cat.totalPages ?? total; const payload: CatalogPayload = { kind: 'catalog_page', house: this.opts.house, event: { ...cat.event, name: cat.event.name || ev.name, url: cat.event.url || ev.url }, page, totalPages: total, items: cat.items }; count++; yield { url, externalId: `event:${cat.event.id}:page:${page}`, kind: cat.event.status === 'past' ? 'sale' : 'auction_lot', engine: r.engine, httpStatus: r.status, payload, fetchedAt: r.fetchedAt }; if (total !== null && page >= total) break; } processed++; if (ev.status === 'past') done.add(ev.id); } const pastPage = Number(ctx.options.cursor?.pastPage ?? 1); await ctx.setCursor({ doneEvents: [...done].slice(-500), pastPage: ctx.options.mode === 'backfill' && events.filter((e) => e.status === 'past').every((e) => done.has(e.id)) ? pastPage + 1 : pastPage, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = CatalogPayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const endDate = bidsquareDate(p.event.endDate); const startDate = bidsquareDate(p.event.startDate); for (const it of p.items) { const slug = this.opts.category(it.title); if (!slug) continue; const g = gradeOf(it.title); const tg = toyGrade(it.title); const attributes = lotAttributes({ categorySlug: slug, name: it.title, year: safeYear(it.title), identifiers: { [this.opts.idKey]: it.itemId }, metadata: { event_id: p.event.id, event_name: p.event.name, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bids: it.bids, ...(tg ? { toy_grade: `${tg.company} ${tg.grade}` } : {}) }, }); const common = { meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade }; const sold = p.event.status === 'past' && /sold/i.test(it.priceLabel ?? '') && it.price !== null && it.price > 0; if (sold) { const saleDate = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate; if (!saleDate) continue; out.push(makeSale({ ...common, price: it.price!, currency: 'USD', saleDate, buyerPremiumIncluded: this.opts.buyerPremiumIncluded, auctionHouse: this.opts.house, lotNumber: it.lotNumber, location: this.opts.location, isBundle: isBundleTitle(it.title) })); } else if (p.event.status !== 'past') { const endsAt = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate; const status = startDate && startDate.getTime() > Date.now() ? 'upcoming' : 'live'; out.push(makeLot({ ...common, auctionHouse: this.opts.house, auctionName: p.event.name, lotNumber: it.lotNumber, startsAt: startDate, endsAt, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: /bid/i.test(it.priceLabel ?? '') ? it.price : null, currency: 'USD', status, location: this.opts.location, confidence: 0.85 })); } } return out; } } export function scpCategory(title: string): string { return sportsCategory(title); } export default (meta: ConnectorMeta) => new BidsquareHouseConnector(meta, { base: 'https://catalogs.scpauctions.com', house: 'SCP Auctions', category: scpCategory, // SCP does not state on the catalog page whether "Sold for" includes the buyer's premium. buyerPremiumIncluded: null, idKey: 'scp_item_id', location: 'US', }); export type { BidsquareCatalog };