TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateWords, makeSale, md, money, splitMarkdownItems, vehicleAttributes } from '../_carlib/index.js';56const BASE = 'https://rmsothebys.com';7const PARSER_VERSION = '1.0.0';89export const AuctionSchema = z.object({ code: z.string(), name: z.string().nullable(), dateText: z.string().nullable() });10export const LotSchema = z.object({ slug: z.string(), url: z.string(), title: z.string(), lotNumber: z.string().nullable(), priceText: z.string().nullable(), status: z.string().nullable(), image: z.string().nullable() });11export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), auction: AuctionSchema, lots: z.array(LotSchema) });12export type PagePayload = z.infer<typeof PagePayloadSchema>;1314/** /results/ markdown: "Auction Name | 13 - 15 August 2026 | [View Results](…/auctions/mo26/lots/)" blocks. */15export function parseResults(markdown: string): z.infer<typeof AuctionSchema>[] {16 const out: z.infer<typeof AuctionSchema>[] = [];17 const re = /\[View Results\]\(https:\/\/rmsothebys\.com\/auctions\/([a-z0-9]+)\/lots\/[^)]*\)/g;18 let m: RegExpExecArray | null;19 while ((m = re.exec(markdown))) {20 const before = markdown.slice(Math.max(0, m.index - 600), m.index);21 const lines = before.split('\n').map((l) => l.trim()).filter((l) => l && !l.startsWith('!') && !/^[‹›]+$/.test(l));22 const dateText = [...lines].reverse().find((l) => /\d{1,2}\s*[-–]?\s*\d{0,2}\s*[A-Z][a-z]+\s+\d{4}|^[A-Z][a-z]+\s+\d{1,2},?\s+\d{4}|Bidding Closes/.test(l)) ?? null;23 const name = [...lines].reverse().find((l) => l !== dateText && !/View Results|Bidding Closes/.test(l) && l.length > 3) ?? null;24 if (!out.some((a) => a.code === m![1])) out.push({ code: m[1]!, name, dateText: dateText?.replace(/^Bidding Closes\s*/i, '') ?? null });25 }26 return out;27}2829export function parseLotsPage(markdown: string, auction: z.infer<typeof AuctionSchema>): PagePayload {30 const chunks = splitMarkdownItems(markdown, /^\[!\[[^\]]*\]\([^)]+\)\]\(https:\/\/rmsothebys\.com\/auctions\/[a-z0-9]+\/lots\//m);31 const lots: z.infer<typeof LotSchema>[] = [];32 for (const c of chunks) {33 const url = c.match(/\]\((https:\/\/rmsothebys\.com\/auctions\/[a-z0-9]+\/lots\/([a-z0-9-]+)\/)\)/);34 if (!url) continue;35 const body = c.match(/\[\*\*[^*]+\*\*\s*\\?\s*\n?([\s\S]*?)\]\(https:\/\/rmsothebys\.com\/auctions\/[a-z0-9]+\/lots\//)?.[1] ?? c;36 const lines = body.split('\n').map((l) => l.replace(/\\+$/, '').replace(/^\\+/, '').trim()).filter(Boolean);37 const title = lines[0] ?? null;38 if (!title) continue;39 const lotLine = lines.find((l) => /^Lot\s+\S+/.test(l)) ?? '';40 const lotNumber = lotLine.match(/^Lot\s+([A-Za-z0-9.]+)/)?.[1] ?? null;41 const priceText = lotLine.match(/\|\s*(.+)$/)?.[1]?.trim() ?? null;42 const status = lines.find((l) => /^(Sold|Not Sold|Withdrawn|Lot Sold|Lot Closed)$/i.test(l)) ?? null;43 lots.push({ slug: url[2]!, url: url[1]!, title: md.clean(title), lotNumber, priceText: priceText && /\d/.test(priceText) ? priceText : null, status, image: md.image(c) });44 }45 return { kind: 'lots_page', auction, lots };46}4748const MEMORABILIA = /\b(sculpture|poster|sign|helmet|model|artwork|painting|trophy|pedal car|neon|literature|memorabilia|watch)\b/i;4950export class RMSothebysConnector extends BaseConnector {51 readonly version = '1.0.0';52 readonly parserVersion = PARSER_VERSION;53 protected override minIntervalMs = 2000;5455 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {56 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 3);57 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);58 const year = ctx.options.mode === 'backfill' ? Number(ctx.options.cursor?.year ?? new Date().getUTCFullYear()) : new Date().getUTCFullYear();59 const listUrl = `${BASE}/results/${ctx.options.mode === 'backfill' ? `?year=${year}` : ''}`;60 await this.throttle();61 const list = await ctx.fetch(listUrl, { waitForMs: 5000, expect: ['title', 'date'], parse: (r) => (r.markdown ? { title: parseResults(r.markdown)[0]?.name ?? null, date: parseResults(r.markdown)[0]?.dateText ?? null } : null), minQuality: 0.3 });62 if (!list.success || !list.markdown) {63 ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);64 return;65 }66 const now = Date.now();67 const skipSealed = this.meta.config.skipSealed !== false;68 const auctions = parseResults(list.markdown).filter((a) => !done.has(a.code)).filter((a) => !(skipSealed && /^sealed/i.test(a.name ?? ''))).filter((a) => {69 const d = dateWords(a.dateText);70 return !d || d.getTime() <= now;71 });72 let count = 0;73 let processed = 0;74 for (const auction of auctions) {75 if (processed >= auctionsPerRun || ctx.signal?.aborted || this.reached(ctx, count)) break;76 const url = `${BASE}/auctions/${auction.code}/lots/`;77 await this.throttle();78 const res = await ctx.fetch(url, {79 waitForMs: 7000,80 expect: ['title', 'price', 'status'],81 parse: (r) => {82 const p = r.markdown ? parseLotsPage(r.markdown, auction) : null;83 const f = p?.lots.find((l) => l.priceText && /sold/i.test(l.status ?? ''));84 return f ? { title: f.title, price: money(f.priceText)?.amount ?? null, status: f.status } : p?.lots.length ? { title: p.lots[0]!.title } : null;85 },86 minQuality: 0.3,87 });88 processed++;89 if (!res.success || !res.markdown) {90 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);91 continue;92 }93 const payload = parseLotsPage(res.markdown, auction);94 if (payload.lots.length === 0) {95 ctx.anomaly('empty_page', url);96 continue;97 }98 count++;99 done.add(auction.code);100 yield { url, externalId: `auction:${auction.code}:first40`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };101 await ctx.setCursor({ doneAuctions: [...done].slice(-200), year: ctx.options.mode === 'backfill' && auctions.every((a) => done.has(a.code)) ? year - 1 : year, updatedAt: new Date().toISOString() });102 }103 }104105 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {106 const p = PagePayloadSchema.parse(raw.payload);107 const saleDate = dateWords(p.auction.dateText);108 if (!saleDate) return [];109 const out: NormalizedSale[] = [];110 for (const lot of p.lots) {111 if (!lot.priceText || !/^(sold|lot sold)$/i.test(lot.status ?? '')) continue;112 const m = money(lot.priceText, 'USD');113 if (!m) continue;114 const memorabilia = MEMORABILIA.test(lot.title) && !/^\d{4}\s/.test(lot.title);115 const attributes = vehicleAttributes(lot.title, { identifiers: { rm_lot: `${p.auction.code}-${lot.slug}` }, metadata: { auction: p.auction.name, auction_code: p.auction.code, auction_dates: p.auction.dateText }, ...(memorabilia ? { categorySlug: 'automotive_memorabilia' as const } : {}) });116 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.code}-${lot.slug}`, rawTitle: lot.title, attributes, price: m.amount, currency: m.currency, saleDate, buyerPremiumIncluded: true, auctionHouse: "RM Sotheby's", lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }));117 }118 return out;119 }120}121122export default (meta: ConnectorMeta) => new RMSothebysConnector(meta);123