TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedAuctionLotSchema, NormalizedSaleSchema, type NormalizedRecord, type CurrencyCode, SUPPORTED_CURRENCIES } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { brandFromSlug, hintFromLabel, isBundleTitle, legoSetNumber, safeYear, slugFromTitle, watchReference } from '../_auction-lib/categories.js';67/**8 * Bonhams — realized prices + upcoming lots from the public, server-rendered Next.js pages.9 * Engine: plain HTTPS (no credits). See meta.json accessNotes.10 */1112const SITE = 'https://www.bonhams.com';1314export const AuctionSummarySchema = z.object({15 id: z.string(),16 title: z.string(),17 slug: z.string(),18 status: z.string().nullable(),19 type: z.string().nullable(),20 departments: z.array(z.string()),21 categories: z.array(z.string()),22 currency: z.string().nullable(),23 country: z.string().nullable(),24 venue: z.string().nullable(),25 start: z.string().nullable(),26 end: z.string().nullable(),27 isEnded: z.boolean(),28 numberOfLots: z.number().nullable(),29});30export type AuctionSummary = z.infer<typeof AuctionSummarySchema>;3132export const LotSchema = z.object({33 lotId: z.string(),34 lotUniqueId: z.string().nullable(),35 lotNo: z.string(),36 title: z.string(),37 heading: z.string().nullable(),38 slug: z.string().nullable(),39 imageUrl: z.string().nullable(),40 estimateLow: z.number().nullable(),41 estimateHigh: z.number().nullable(),42 hammerPrice: z.number().nullable(),43 hammerPremium: z.number().nullable(),44 startingBid: z.number().nullable(),45 currency: z.string().nullable(),46 status: z.string().nullable(),47 hammerTime: z.string().nullable(),48 endDate: z.string().nullable(),49 department: z.string().nullable(),50 categories: z.array(z.string()),51 isEnded: z.boolean(),52 isWithoutReserve: z.boolean().nullable(),53});54export type Lot = z.infer<typeof LotSchema>;5556export const PagePayloadSchema = z.object({57 kind: z.literal('auction_lots'),58 auction: AuctionSummarySchema,59 page: z.number().int(),60 nbHits: z.number().nullable(),61 lots: z.array(LotSchema),62});63export type PagePayload = z.infer<typeof PagePayloadSchema>;6465const ConfigSchema = z.object({66 departments: z.array(z.string()).default([]),67 maxResultsPages: z.number().int().min(1).default(20),68 maxAuctionsPerRun: z.number().int().min(1).default(40),69 upcomingAuctionsPerRun: z.number().int().min(0).default(12),70 lotsPerPage: z.number().int().default(48),71});7273type NextData = { props?: { pageProps?: Record<string, unknown> } };7475export function extractNextData(html: string): NextData | null {76 const m = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);77 if (!m) return null;78 try {79 return JSON.parse(m[1]!) as NextData;80 } catch {81 return null;82 }83}8485function str(v: unknown): string | null {86 return typeof v === 'string' && v.length ? v : v === null || v === undefined ? null : typeof v === 'number' ? String(v) : null;87}88function numOrNull(v: unknown): number | null {89 return typeof v === 'number' && Number.isFinite(v) ? v : null;90}9192/** Summaries from a results/upcoming listing page (pagesOfAuctions) or from an auction page (auction + lots). */93export function parseAuctionList(html: string): { auctions: AuctionSummary[]; nbHits: number | null } {94 const nd = extractNextData(html);95 const pp = nd?.props?.pageProps ?? {};96 const pages = (pp.pagesOfAuctions as unknown[][] | undefined) ?? [];97 const auctions: AuctionSummary[] = [];98 for (const page of pages) {99 for (const a of page as Array<Record<string, any>>) {100 const dates = a.dates ?? {};101 auctions.push(102 AuctionSummarySchema.parse({103 id: String(a.id),104 title: a.auctionTitle ?? a.auctionHeading ?? '',105 slug: a.slug ?? '',106 status: str(a.auctionStatus),107 type: str(a.auctionType),108 departments: ((a.departments as Array<{ name?: string }> | undefined) ?? []).map((d) => d.name ?? '').filter(Boolean),109 categories: ((a.categories as Array<{ name?: string }> | undefined) ?? []).map((d) => d.name ?? '').filter(Boolean),110 currency: str(a.currency?.iso_code),111 country: str(a.country?.code),112 venue: str(a.venue ?? a.location?.name),113 start: str(dates.start?.datetime),114 end: str(dates.end?.datetime ?? a.hammerTime?.datetime),115 isEnded: Boolean(a.flags?.isAuctionEnded),116 numberOfLots: numOrNull(a.numberOfLots ?? a.number_of_lots),117 }),118 );119 }120 }121 return { auctions, nbHits: numOrNull(pp.nbHits) };122}123124/** Lots + auction metadata from an auction page (or its _next/data JSON pageProps). */125export function parseAuctionPage(html: string, fallback?: Partial<AuctionSummary>): { auction: AuctionSummary; lots: Lot[]; nbHits: number | null } | null {126 const nd = extractNextData(html);127 const pp = nd?.props?.pageProps as Record<string, any> | undefined;128 if (!pp?.lotData) return null;129 const a = pp.auction ?? {};130 const rawLots = (pp.lotData.auctionLots as Array<Record<string, any>> | undefined) ?? [];131 const first = rawLots[0];132 const auction = AuctionSummarySchema.parse({133 id: String(first?.auctionId ?? fallback?.id ?? a.iSaleNo ?? ''),134 title: a.sSaleName ?? fallback?.title ?? '',135 slug: a.slug ?? fallback?.slug ?? '',136 status: str(first?.auctionStatus) ?? fallback?.status ?? null,137 type: str(first?.auctionType) ?? str(a.sSaleType) ?? fallback?.type ?? null,138 departments: ((a.departments as Array<{ sDepartmentName?: string }> | undefined) ?? []).map((d) => d.sDepartmentName ?? '').filter(Boolean),139 categories: fallback?.categories ?? [],140 currency: str(first?.currency?.iso_code) ?? fallback?.currency ?? null,141 country: str(first?.country?.code) ?? fallback?.country ?? null,142 venue: str(a.sVenue) ?? fallback?.venue ?? null,143 start: str(a.dates?.start?.[0]?.date?.datetime) ?? fallback?.start ?? null,144 end: str(a.dates?.end?.datetime) ?? str(first?.auctionEndDate?.datetime) ?? fallback?.end ?? null,145 isEnded: Boolean(first?.flags?.isAuctionEnded ?? fallback?.isEnded ?? false),146 numberOfLots: numOrNull(a.number_of_lots) ?? fallback?.numberOfLots ?? null,147 });148 const lots: Lot[] = rawLots.map((l) =>149 LotSchema.parse({150 lotId: String(l.lotId ?? l.id ?? ''),151 lotUniqueId: str(l.lotUniqueId),152 lotNo: String(l.lotNo?.full ?? l.lotNo?.number ?? l.lotId ?? ''),153 title: String(l.title ?? l.image?.caption ?? '').replace(/\s+/g, ' ').trim(),154 heading: str(l.heading) || null,155 slug: str(l.slug),156 imageUrl: str(l.image?.url),157 estimateLow: numOrNull(l.price?.estimateLow),158 estimateHigh: numOrNull(l.price?.estimateHigh),159 hammerPrice: numOrNull(l.price?.hammerPrice),160 hammerPremium: numOrNull(l.price?.hammerPremium),161 startingBid: numOrNull(l.price?.startingBidAmount),162 currency: str(l.currency?.iso_code),163 status: str(l.status),164 hammerTime: str(l.hammerTime?.datetime),165 endDate: str(l.auctionEndDate?.datetime),166 department: str(l.department?.name),167 categories: ((l.categories as Array<{ name?: string }> | undefined) ?? []).map((c) => c.name ?? '').filter(Boolean),168 isEnded: Boolean(l.flags?.isAuctionEnded),169 isWithoutReserve: typeof l.flags?.isWithoutReserve === 'boolean' ? l.flags.isWithoutReserve : null,170 }),171 );172 return { auction, lots, nbHits: numOrNull(pp.lotData.nbHits) };173}174175export function lotUrl(auctionId: string, lotNo: string, slug: string | null): string {176 return `${SITE}/auction/${auctionId}/lot/${lotNo}/${slug ? `${slug}/` : ''}`;177}178export function auctionPageUrl(a: { id: string; slug: string }, page: number): string {179 return `${SITE}/auction/${a.id}/${a.slug}/${page > 1 ? `?page=${page}` : ''}`;180}181182function currency(code: string | null | undefined): CurrencyCode | null {183 return code && (SUPPORTED_CURRENCIES as readonly string[]).includes(code) ? (code as CurrencyCode) : null;184}185186function parseDate(s: string | null | undefined): Date | null {187 if (!s) return null;188 const d = new Date(s);189 return Number.isNaN(d.getTime()) ? null : d;190}191192export default function createConnector(meta: ConnectorMeta) {193 return new BonhamsConnector(meta);194}195196export class BonhamsConnector extends BaseConnector {197 readonly version = '1.0.0';198 readonly parserVersion = '1.0.0';199 override readonly urlPatterns = [/^https?:\/\/(www\.)?bonhams\.com\/auction\/\d+\//i];200 protected override minIntervalMs = 1500;201 private readonly config = ConfigSchema.parse(this.meta.config ?? {});202203 private wanted(a: AuctionSummary): boolean {204 if (!this.config.departments.length) return true;205 return a.departments.some((d) => this.config.departments.includes(d));206 }207208 private async page(ctx: CrawlContext, url: string): Promise<string | null> {209 await this.throttle();210 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, headers: { accept: 'text/html' } });211 if (!res.success || !res.html) {212 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);213 return null;214 }215 return res.html;216 }217218 /** Iterate listing pages (results or upcoming) yielding whitelisted auctions until `stop` says so. */219 private async *listAuctions(ctx: CrawlContext, path: string, maxPages: number, stop: (a: AuctionSummary) => boolean): AsyncIterable<AuctionSummary> {220 for (let page = 1; page <= maxPages; page++) {221 const html = await this.page(ctx, `${SITE}${path}${page > 1 ? `?page=${page}` : ''}`);222 if (!html) return;223 const { auctions } = parseAuctionList(html);224 if (!auctions.length) {225 if (page === 1) ctx.anomaly('empty_page', `${path}: no auctions parsed (redesign?)`);226 return;227 }228 for (const a of auctions) {229 if (stop(a)) return;230 if (this.wanted(a)) yield a;231 }232 }233 }234235 private async *crawlAuction(ctx: CrawlContext, a: AuctionSummary, kind: 'sale' | 'auction_lot', maxPages = 60): AsyncIterable<RawRecordInput> {236 for (let page = 1; page <= maxPages; page++) {237 const url = auctionPageUrl(a, page);238 const html = await this.page(ctx, url);239 if (!html) return;240 const parsed = parseAuctionPage(html, a);241 if (!parsed) {242 ctx.anomaly('parse_failure', `${url}: no lotData in __NEXT_DATA__`);243 return;244 }245 if (!parsed.lots.length) return;246 const payload: PagePayload = { kind: 'auction_lots', auction: { ...parsed.auction, categories: a.categories }, page, nbHits: parsed.nbHits, lots: parsed.lots };247 yield { url, externalId: `${a.id}#${page}`, kind, engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };248 if (parsed.nbHits !== null && page * this.config.lotsPerPage >= parsed.nbHits) return;249 if (ctx.signal?.aborted) return;250 }251 }252253 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {254 const cursor = { ...(ctx.options.cursor ?? {}) } as { lastEnd?: string; done?: string[] };255 const done = new Set(cursor.done ?? []);256 const probe = ctx.options.mode === 'probe';257 const backfill = ctx.options.mode === 'backfill';258 const maxAuctions = probe ? 1 : this.config.maxAuctionsPerRun;259 let yielded = 0;260 let newestEnd: string | undefined = cursor.lastEnd;261 let count = 0;262263 // 1. Past results (sold)264 const stop = (a: AuctionSummary) => !backfill && !!cursor.lastEnd && !!a.end && a.end < cursor.lastEnd && !probe;265 for await (const a of this.listAuctions(ctx, '/auctions/results/', probe ? 1 : this.config.maxResultsPages, stop)) {266 if (done.has(a.id)) continue;267 if (count >= maxAuctions) break;268 count++;269 for await (const raw of this.crawlAuction(ctx, a, 'sale', probe ? 1 : 60)) {270 yield raw;271 yielded += (raw.payload as PagePayload).lots.length;272 if (this.reached(ctx, yielded)) return;273 }274 done.add(a.id);275 if (a.end && (!newestEnd || a.end > newestEnd)) newestEnd = a.end;276 cursor.done = [...done].slice(-500);277 if (!backfill) cursor.lastEnd = newestEnd;278 await ctx.setCursor(cursor);279 }280281 // 2. Upcoming / live lots for the auction calendar (incremental runs only)282 if (!probe && !backfill && this.config.upcomingAuctionsPerRun > 0) {283 let up = 0;284 for await (const a of this.listAuctions(ctx, '/auctions/upcoming/', 3, () => false)) {285 if (a.type === 'EXHIBITION') continue;286 if (up++ >= this.config.upcomingAuctionsPerRun) break;287 for await (const raw of this.crawlAuction(ctx, a, 'auction_lot', 10)) yield raw;288 }289 }290 }291292 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {293 const m = url.match(/\/auction\/(\d+)\/(?:lot\/(\d+[A-Za-z]?)\/)?([^/?#]*)/);294 if (!m) return [];295 const auctionId = m[1]!;296 const lotNo = m[2];297 // The auction page lists lots 48 at a time; find the page containing the lot number when given.298 const first = await this.page(ctx, `${SITE}/auction/${auctionId}/`);299 if (!first) return [];300 const parsed = parseAuctionPage(first);301 if (!parsed) return [];302 let lots = parsed.lots;303 if (lotNo && !lots.some((l) => l.lotNo === lotNo) && parsed.nbHits) {304 const pages = Math.ceil(parsed.nbHits / this.config.lotsPerPage);305 for (let p = 2; p <= pages; p++) {306 const html = await this.page(ctx, auctionPageUrl({ id: auctionId, slug: parsed.auction.slug }, p));307 const pg = html ? parseAuctionPage(html, parsed.auction) : null;308 if (pg?.lots.some((l) => l.lotNo === lotNo)) {309 lots = pg.lots;310 break;311 }312 }313 }314 if (lotNo) lots = lots.filter((l) => l.lotNo === lotNo);315 const payload: PagePayload = { kind: 'auction_lots', auction: parsed.auction, page: 0, nbHits: parsed.nbHits, lots };316 return [{ url, externalId: lotNo ? `${auctionId}-${lotNo}` : auctionId, kind: parsed.auction.isEnded ? 'sale' : 'auction_lot', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() }];317 }318319 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {320 const page = PagePayloadSchema.parse(raw.payload);321 const out: NormalizedRecord[] = [];322 const a = page.auction;323 const observedAt = raw.fetchedAt;324 for (const lot of page.lots) {325 const dept = lot.department ?? a.departments[0] ?? null;326 // A lot's own department is authoritative; fall back to the sale's departments when the lot has none.327 if (this.config.departments.length && (dept ? !this.config.departments.includes(dept) : !a.departments.some((d) => this.config.departments.includes(d)))) continue;328 const hint = hintFromLabel(dept);329 const categorySlug = slugFromTitle(lot.title, hint);330 if (!categorySlug) continue; // unmapped department/title → skip rather than guess331 const cur = currency(lot.currency ?? a.currency);332 if (!cur) continue;333 const grade = parseGradeFromTitle(lot.title);334 const brand = brandFromSlug(categorySlug, lot.title);335 const reference = categorySlug.endsWith('watches') || ['rolex', 'patek_philippe', 'audemars_piguet', 'omega'].includes(categorySlug) ? watchReference(lot.title) : null;336 const identifiers: Record<string, string> = { bonhams_lot: `${a.id}-${lot.lotNo}` };337 if (lot.lotUniqueId) identifiers.bonhams_lot_unique_id = lot.lotUniqueId;338 if (reference) identifiers.reference = reference;339 if (categorySlug === 'lego_sets') {340 const n = legoSetNumber(lot.title);341 if (n) identifiers.lego_set_number = n;342 }343 const base = {344 connectorId: this.meta.id,345 sourceId: this.meta.sourceId,346 sourceUrl: lotUrl(a.id, lot.lotNo, lot.slug),347 externalId: `${a.id}-${lot.lotNo}`,348 rawTitle: lot.title,349 description: lot.heading,350 imageUrls: lot.imageUrl ? [lot.imageUrl] : [],351 attributes: {352 categorySlug,353 name: lot.title,354 brand,355 reference,356 year: safeYear(lot.title),357 country: a.country,358 identifiers,359 metadata: { department: dept, auction_id: a.id, auction_title: a.title, auction_type: a.type, venue: a.venue, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, hammer_price: lot.hammerPrice, without_reserve: lot.isWithoutReserve },360 },361 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },362 condition: {},363 observedAt,364 parserVersion: this.parserVersion,365 };366 const saleDate = parseDate(lot.hammerTime ?? lot.endDate ?? a.end);367 const sold = lot.status === 'SOLD' && (lot.hammerPremium ?? lot.hammerPrice ?? 0) > 0;368 const endDate = parseDate(lot.endDate ?? a.end);369 const ended = sold || lot.isEnded || a.isEnded || lot.status === 'UNSOLD' || lot.status === 'WITHDRAWN' || (endDate !== null && endDate.getTime() < Date.now());370 if (ended) {371 // Only published results (status SOLD with a price) become sales; unsold/withdrawn lots are skipped.372 if (!sold || !saleDate || saleDate.getTime() > Date.now() + 86_400_000) continue;373 const price = lot.hammerPremium ?? lot.hammerPrice!;374 out.push(375 NormalizedSaleSchema.parse({376 ...base,377 kind: 'sale',378 confidence: 0.9,379 saleType: 'auction',380 saleDate,381 price,382 currency: cur,383 buyerPremiumIncluded: lot.hammerPremium !== null,384 quantity: 1,385 isBundle: isBundleTitle(lot.title),386 location: a.venue ?? a.country,387 auctionHouse: 'Bonhams',388 lotNumber: lot.lotNo,389 }),390 );391 } else {392 const endsAt = parseDate(lot.endDate ?? a.end);393 const startsAt = parseDate(a.start);394 const now = Date.now();395 const status = startsAt && startsAt.getTime() <= now ? 'live' : 'upcoming';396 out.push(397 NormalizedAuctionLotSchema.parse({398 ...base,399 kind: 'auction_lot',400 confidence: 0.85,401 auctionHouse: 'Bonhams',402 auctionName: a.title,403 lotNumber: lot.lotNo,404 startsAt,405 endsAt,406 estimateLow: lot.estimateLow,407 estimateHigh: lot.estimateHigh,408 currentBid: null,409 currency: cur,410 status,411 location: a.venue ?? a.country,412 }),413 );414 }415 }416 return out;417 }418}419