TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedAuctionLotSchema, NormalizedSaleSchema, SUPPORTED_CURRENCIES, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { brandFromSlug, hintFromLabel, isBundleTitle, legoSetNumber, safeYear, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js';67/**8 * Sotheby's — results (price incl. premium) and upcoming lots from the public auction pages and the9 * page's own GraphQL lot-card pagination. Engine: plain HTTPS (+ Firecrawl for link discovery).10 */1112const SITE = 'https://www.sothebys.com';13const GRAPHQL = 'https://clientapi.prod.sothelabs.com/graphql';1415export const AuctionSchema = z.object({16 auctionId: z.string(),17 url: z.string(),18 title: z.string(),19 saleNumber: z.string().nullable(),20 state: z.string().nullable(), // Opened | Closed | Published…21 type: z.string().nullable(), // Timed | Live22 departments: z.array(z.string()),23 currency: z.string().nullable(),24 location: z.string().nullable(),25 startsAt: z.string().nullable(),26 endsAt: z.string().nullable(),27 closedAt: z.string().nullable(),28 totalLots: z.number().nullable(),29});30export type Auction = z.infer<typeof AuctionSchema>;3132export const LotSchema = z.object({33 lotId: z.string(),34 lotNumber: z.string().nullable(),35 title: z.string(),36 creators: z.string().nullable(),37 slug: z.string().nullable(),38 estimateLow: z.number().nullable(),39 estimateHigh: z.number().nullable(),40 isClosed: z.boolean().nullable(),41 closingTime: z.string().nullable(),42 currentBid: z.number().nullable(),43 bidCurrency: z.string().nullable(),44 isSold: z.boolean().nullable(),45 finalPrice: z.number().nullable(),46 finalCurrency: z.string().nullable(),47 numberOfBids: z.number().nullable(),48 imageUrl: z.string().nullable(),49 withdrawn: z.boolean(),50});51export type Lot = z.infer<typeof LotSchema>;5253export const PagePayloadSchema = z.object({54 kind: z.literal('auction_page'),55 auction: AuctionSchema,56 offset: z.number().int(),57 lots: z.array(LotSchema),58});59export type PagePayload = z.infer<typeof PagePayloadSchema>;6061const ConfigSchema = z.object({62 seeds: z.array(z.string()).default([]),63 discoveryPages: z.array(z.string()).default([]),64 departments: z.array(z.string()).default([]),65 maxAuctionsPerRun: z.number().int().default(25),66 pageSize: z.number().int().min(1).max(48).default(48),67});6869const LOT_QUERY = `query LotCardsFilterByPaginated($id: String!, $limit: Int, $offset: Int) {70 auction(id: $id, language: ENGLISH) {71 id72 lotCards: lotCardsConnection(offset: $offset, limit: $limit, filter: ALL) {73 totalCount74 hasNextPage75 lots {76 lotId77 title78 creatorsDisplayTitle79 lotNumber { ... on VisibleLotNumber { lotDisplayNumber } }80 slug { lotSlug }81 withdrawnState { state }82 estimateV2 { ... on LowHighEstimateV2 { lowEstimate { amount } highEstimate { amount } } }83 bidState {84 isClosed85 closingTime86 numberOfBids87 currentBidV2 { amount currency }88 sold { __typename ... on ResultVisible { isSold premiums { finalPriceV2 { amount currency } } } }89 }90 media(imageSizes: [Small]) { images { renditions { url } } }91 }92 }93 }94}`;9596function num(v: unknown): number | null {97 if (v === null || v === undefined || v === '') return null;98 const n = typeof v === 'number' ? v : Number.parseFloat(String(v));99 return Number.isFinite(n) ? n : null;100}101102/** Build a Lot from a GraphQL lot card or from the SSR Apollo cache (refs resolved by caller). */103function lotFrom(card: Record<string, any>, bidState: Record<string, any> | null): Lot {104 const est = card.estimateV2 ?? {};105 const sold = bidState?.sold ?? {};106 const fp = sold.premiums?.finalPriceV2 ?? null;107 const cb = bidState?.currentBidV2 ?? null;108 const mediaKey = Object.keys(card).find((k) => k.startsWith('media'));109 const img = mediaKey ? card[mediaKey]?.images?.[0]?.renditions?.[0]?.url ?? null : null;110 return LotSchema.parse({111 lotId: String(card.lotId),112 lotNumber: card.lotNumber?.lotDisplayNumber ?? null,113 title: String(card.title ?? ''),114 creators: card.creatorsDisplayTitle ?? null,115 slug: card.slug?.lotSlug ?? null,116 estimateLow: num(est.lowEstimate?.amount),117 estimateHigh: num(est.highEstimate?.amount),118 isClosed: typeof bidState?.isClosed === 'boolean' ? bidState.isClosed : null,119 closingTime: bidState?.closingTime ?? null,120 currentBid: num(cb?.amount),121 bidCurrency: cb?.currency ?? null,122 isSold: typeof sold.isSold === 'boolean' ? sold.isSold : null,123 finalPrice: num(fp?.amount),124 finalCurrency: fp?.currency ?? null,125 numberOfBids: num(bidState?.numberOfBids),126 imageUrl: typeof img === 'string' ? img : null,127 withdrawn: card.withdrawnState?.state ? card.withdrawnState.state !== 'NotAffected' : false,128 });129}130131/** Parse an SSR auction page: auction + first lot cards from the Apollo cache. */132export function parseAuctionPage(html: string, url: string): { auction: Auction; lots: Lot[] } | null {133 const m = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);134 if (!m) return null;135 let cache: Record<string, any>;136 let pp: Record<string, any>;137 try {138 pp = (JSON.parse(m[1]!) as { props: { pageProps: Record<string, any> } }).props.pageProps;139 cache = pp.apolloCache ?? {};140 } catch {141 return null;142 }143 const aKey = Object.keys(cache).find((k) => k.startsWith('Auction:'));144 if (!aKey) return null;145 const a = cache[aKey];146 const dates = a.dates ?? {};147 const cards = Object.keys(cache)148 .filter((k) => k.startsWith('LotCard:'))149 .map((k) => cache[k])150 .filter((c) => !c.auction || c.auction.auctionId === a.auctionId || c.auction.sapSaleNumber === a.sapSaleNumber);151 const lots = cards.map((c) => lotFrom(c, c.bidState?.__ref ? cache[c.bidState.__ref] ?? null : c.bidState ?? null));152 const currency = a.currencyV2 ?? a.currency ?? cards[0]?.auction?.currency ?? lots.find((l) => l.finalCurrency)?.finalCurrency ?? null;153 const auction = AuctionSchema.parse({154 auctionId: String(a.auctionId),155 url,156 title: String(a.title ?? ''),157 saleNumber: a.sapSaleNumber ?? null,158 state: a.state ?? null,159 type: a.type ?? null,160 departments: Array.isArray(a.departmentNames) ? a.departmentNames.map((d: string) => d.trim()) : [],161 currency,162 location: a.locationV2?.name ?? null,163 startsAt: dates.goesLive ?? dates.acceptsBids ?? null,164 endsAt: dates.startsToClose ?? dates.goesLive ?? null,165 closedAt: dates.closed ?? null,166 totalLots: num(pp.totalLotCount),167 });168 return { auction, lots };169}170171export function parseGraphqlLots(json: any): { lots: Lot[]; hasNextPage: boolean; totalCount: number | null } | null {172 const conn = json?.data?.auction?.lotCards;173 if (!conn) return null;174 return { lots: (conn.lots ?? []).map((c: Record<string, any>) => lotFrom(c, c.bidState ?? null)), hasNextPage: Boolean(conn.hasNextPage), totalCount: num(conn.totalCount) };175}176177export function auctionLinks(html: string): string[] {178 return [...new Set([...html.matchAll(/https?:\/\/www\.sothebys\.com\/en\/buy\/auction\/(20\d{2})\/([a-z0-9-]+)/g)].map((m) => `${SITE}/en/buy/auction/${m[1]}/${m[2]}`))];179}180181export function lotUrl(auctionUrl: string, lotSlug: string | null, lotId: string): string {182 return lotSlug ? `${auctionUrl}/${lotSlug}` : `${auctionUrl}?lotId=${lotId}`;183}184185function currency(code: string | null | undefined): CurrencyCode | null {186 return code && (SUPPORTED_CURRENCIES as readonly string[]).includes(code) ? (code as CurrencyCode) : null;187}188189/** Ordered hints for an auction: a single-department sale is a strong hint; a mixed "Arcade"-style sale is not. */190export function hintsForAuction(a: Auction): DeptHint[] {191 const hints = a.departments.map((d) => hintFromLabel(d)).filter((h) => h !== 'unknown');192 const titleHint = hintFromLabel(a.title);193 // Mixed multi-department sales (e.g. "Arcade"): only title keywords are trusted; no department guess.194 if (a.departments.length > 3) return ['unknown', ...(titleHint !== 'unknown' ? [titleHint] : [])];195 return [...(hints.length ? hints : []), ...(titleHint !== 'unknown' ? [titleHint] : []), 'unknown'];196}197198export function classifyTitle(title: string, hints: DeptHint[]): string | null {199 for (const h of hints) {200 const slug = slugFromTitle(title, h);201 if (slug) return slug;202 }203 return null;204}205206function iso(d: string | null | undefined): Date | null {207 if (!d) return null;208 const x = new Date(d);209 return Number.isNaN(x.getTime()) ? null : x;210}211212export default function createConnector(meta: ConnectorMeta) {213 return new SothebysConnector(meta);214}215216export class SothebysConnector extends BaseConnector {217 readonly version = '1.0.0';218 readonly parserVersion = '1.0.0';219 override readonly urlPatterns = [/^https?:\/\/(www\.)?sothebys\.com\/en\/buy\/auction\/20\d{2}\/[a-z0-9-]+/i];220 protected override minIntervalMs = 1500;221 private readonly config = ConfigSchema.parse(this.meta.config ?? {});222223 private async page(ctx: CrawlContext, url: string): Promise<string | null> {224 await this.throttle();225 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, headers: { accept: 'text/html' } });226 if (!res.success || !res.html) {227 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);228 return null;229 }230 return res.html;231 }232233 private async graphql(ctx: CrawlContext, auctionId: string, offset: number): Promise<ReturnType<typeof parseGraphqlLots>> {234 await this.throttle();235 const res = await ctx.fetch(GRAPHQL, {236 engines: ['api'],237 method: 'POST',238 body: { operationName: 'LotCardsFilterByPaginated', query: LOT_QUERY, variables: { id: auctionId, limit: this.config.pageSize, offset } },239 headers: { origin: SITE, referer: `${SITE}/`, accept: 'application/json' },240 responseType: 'json',241 minQuality: 0,242 });243 if (!res.success || !res.json) {244 ctx.anomaly('page_fetch_failed', `graphql ${auctionId}@${offset}: ${res.error ?? res.httpStatus}`);245 return null;246 }247 const parsed = parseGraphqlLots(res.json);248 if (!parsed) ctx.anomaly('parse_failure', `graphql ${auctionId}@${offset}: ${JSON.stringify((res.json as { errors?: unknown }).errors ?? '').slice(0, 200)}`);249 return parsed;250 }251252 private async discover(ctx: CrawlContext): Promise<string[]> {253 const found = new Set<string>(this.config.seeds);254 for (const page of this.config.discoveryPages) {255 const res = await ctx.fetch(page, { engines: ['firecrawl'], waitForMs: 4000, minQuality: 0 });256 if (!res.success || !res.html) {257 ctx.anomaly('page_fetch_failed', `${page}: ${res.error ?? res.httpStatus}`);258 continue;259 }260 for (const u of auctionLinks(res.html)) found.add(u);261 }262 return [...found];263 }264265 private async *crawlAuction(ctx: CrawlContext, url: string, probe: boolean): AsyncIterable<RawRecordInput> {266 const html = await this.page(ctx, url);267 if (!html) return;268 const parsed = parseAuctionPage(html, url);269 if (!parsed) {270 ctx.anomaly('parse_failure', `${url}: no Auction in __NEXT_DATA__`);271 return;272 }273 const a = parsed.auction;274 if (this.config.departments.length && !a.departments.some((d) => this.config.departments.includes(d))) return;275 const kind = a.state === 'Closed' ? 'sale' : 'auction_lot';276 const first: PagePayload = { kind: 'auction_page', auction: a, offset: 0, lots: parsed.lots };277 yield { url, externalId: `${a.auctionId}#0`, kind, engine: 'api', httpStatus: 200, payload: first, fetchedAt: new Date() };278 if (probe) return;279 let offset = parsed.lots.length;280 const total = a.totalLots ?? Infinity;281 for (let guard = 0; offset < total && guard < 60; guard++) {282 const gl = await this.graphql(ctx, a.auctionId, offset);283 if (!gl || !gl.lots.length) break;284 const payload: PagePayload = { kind: 'auction_page', auction: a, offset, lots: gl.lots };285 yield { url: `${url}#offset=${offset}`, externalId: `${a.auctionId}#${offset}`, kind, engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };286 offset += gl.lots.length;287 if (!gl.hasNextPage) break;288 }289 }290291 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {292 const cursor = { auctions: {}, ...(ctx.options.cursor ?? {}) } as { auctions: Record<string, { state: string | null; checkedAt: string }> };293 const probe = ctx.options.mode === 'probe';294 const urls = ctx.options.seeds?.length ? ctx.options.seeds : await this.discover(ctx);295 // Re-check auctions previously seen open (their results become visible after close).296 for (const [u, s] of Object.entries(cursor.auctions)) if (s.state !== 'Closed' && !urls.includes(u)) urls.push(u);297 let n = 0;298 for (const url of urls) {299 if (ctx.signal?.aborted) return;300 const prev = cursor.auctions[url];301 if (prev?.state === 'Closed' && ctx.options.mode !== 'backfill') continue;302 if (n++ >= (probe ? 1 : this.config.maxAuctionsPerRun)) break;303 let state: string | null = prev?.state ?? null;304 for await (const raw of this.crawlAuction(ctx, url, probe)) {305 state = (raw.payload as PagePayload).auction.state;306 yield raw;307 }308 cursor.auctions[url] = { state, checkedAt: new Date().toISOString() };309 const entries = Object.entries(cursor.auctions);310 if (entries.length > 400) cursor.auctions = Object.fromEntries(entries.slice(-400));311 await ctx.setCursor(cursor);312 }313 }314315 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {316 const m = url.match(/^(https?:\/\/(?:www\.)?sothebys\.com\/en\/buy\/auction\/20\d{2}\/[a-z0-9-]+)(?:\/([a-z0-9-]+))?/i);317 if (!m) return [];318 const out: RawRecordInput[] = [];319 for await (const raw of this.crawlAuction(ctx, m[1]!, true)) {320 if (m[2]) {321 const p = raw.payload as PagePayload;322 p.lots = p.lots.filter((l) => l.slug === m[2]);323 }324 out.push(raw);325 }326 return out;327 }328329 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {330 const page = PagePayloadSchema.parse(raw.payload);331 const a = page.auction;332 const hints = hintsForAuction(a);333 const out: NormalizedRecord[] = [];334 for (const lot of page.lots) {335 if (lot.withdrawn) continue;336 const title = lot.creators && !lot.title.toLowerCase().includes(lot.creators.toLowerCase()) ? `${lot.creators} — ${lot.title}` : lot.title;337 const categorySlug = classifyTitle(title, hints);338 if (!categorySlug) continue;339 const cur = currency(lot.finalCurrency ?? lot.bidCurrency ?? a.currency);340 if (!cur) continue;341 const grade = parseGradeFromTitle(title);342 const isWatch = ['rolex', 'patek_philippe', 'audemars_piguet', 'omega', 'other_watches'].includes(categorySlug);343 const reference = isWatch ? watchReference(title) : null;344 const identifiers: Record<string, string> = { sothebys_lot_id: lot.lotId };345 if (a.saleNumber) identifiers.sothebys_sale_number = a.saleNumber;346 if (reference) identifiers.reference = reference;347 if (categorySlug === 'lego_sets') {348 const n = legoSetNumber(title);349 if (n) identifiers.lego_set_number = n;350 }351 const base = {352 connectorId: this.meta.id,353 sourceId: this.meta.sourceId,354 sourceUrl: lotUrl(a.url, lot.slug, lot.lotId),355 externalId: lot.lotId,356 rawTitle: title,357 description: null,358 imageUrls: lot.imageUrl ? [lot.imageUrl] : [],359 attributes: {360 categorySlug,361 name: lot.title,362 brand: brandFromSlug(categorySlug, title) ?? (isWatch ? lot.creators : null),363 reference,364 year: safeYear(title),365 identifiers,366 metadata: { auction_id: a.auctionId, sale_number: a.saleNumber, auction_title: a.title, auction_type: a.type, departments: a.departments, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, hammer_price: lot.currentBid, bids: lot.numberOfBids },367 },368 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },369 condition: {},370 observedAt: raw.fetchedAt,371 parserVersion: this.parserVersion,372 };373 const closed = a.state === 'Closed' || lot.isClosed === true;374 if (closed) {375 if (!lot.isSold || !lot.finalPrice || lot.finalPrice <= 0) continue;376 const saleDate = iso(lot.closingTime) ?? iso(a.closedAt) ?? iso(a.endsAt);377 if (!saleDate || saleDate.getTime() > Date.now() + 86_400_000) continue;378 out.push(379 NormalizedSaleSchema.parse({380 ...base,381 kind: 'sale',382 confidence: 0.9,383 saleType: 'auction',384 saleDate,385 price: lot.finalPrice,386 currency: cur,387 buyerPremiumIncluded: true,388 quantity: 1,389 isBundle: isBundleTitle(title),390 location: a.location,391 auctionHouse: "Sotheby's",392 lotNumber: lot.lotNumber,393 }),394 );395 } else {396 const startsAt = iso(a.startsAt);397 out.push(398 NormalizedAuctionLotSchema.parse({399 ...base,400 kind: 'auction_lot',401 confidence: 0.85,402 auctionHouse: "Sotheby's",403 auctionName: a.title,404 lotNumber: lot.lotNumber,405 startsAt,406 endsAt: iso(lot.closingTime) ?? iso(a.endsAt),407 estimateLow: lot.estimateLow,408 estimateHigh: lot.estimateHigh,409 currentBid: lot.currentBid,410 currency: cur,411 status: a.state === 'Opened' && startsAt && startsAt.getTime() <= Date.now() ? 'live' : 'upcoming',412 location: a.location,413 }),414 );415 }416 }417 return out;418 }419}420