TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedSaleSchema, SUPPORTED_CURRENCIES, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared';4import { parseGradeFromTitle } from '@rareindex/taxonomy';5import { brandFromSlug, hintFromLabel, isBundleTitle, legoSetNumber, safeYear, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js';67/**8 * Christie's — prices realised from the public discovery-website JSON endpoints (§110).9 * Engine: plain HTTPS. See meta.json accessNotes.10 */1112const SITE = 'https://www.christies.com';13const RESULTS = `${SITE}/api/discoverywebsite/auctioncalendar/auctionresults`;14const LOTSEARCH = `${SITE}/api/discoverywebsite/auctionpages/lotsearch`;15/** christies.com's edge silently drops requests whose User-Agent contains a URL or '@'; we still identify ourselves honestly. */16const UA = 'RareIndex/0.1 (market data research; contact data at rareindex.io)';1718export const SaleSchema = z.object({19 saleId: z.string(),20 saleNumber: z.string(),21 title: z.string(),22 subtitle: z.string().nullable(),23 eventType: z.string().nullable(), // Live | Online24 location: z.string().nullable(),25 startDate: z.string().nullable(),26 endDate: z.string().nullable(),27 landingUrl: z.string().nullable(),28 categoryLabels: z.array(z.string()),29 saleTotalText: z.string().nullable(),30});31export type Sale = z.infer<typeof SaleSchema>;3233export const LotSchema = z.object({34 objectId: z.string(),35 lotNumber: z.string(),36 titlePrimary: z.string(),37 titleSecondary: z.string().nullable(),38 titleTertiary: z.string().nullable(),39 description: z.string().nullable(),40 url: z.string().nullable(),41 imageUrl: z.string().nullable(),42 estimateLow: z.number().nullable(),43 estimateHigh: z.number().nullable(),44 estimateText: z.string().nullable(),45 priceRealised: z.number().nullable(),46 priceRealisedText: z.string().nullable(),47 startDate: z.string().nullable(),48 endDate: z.string().nullable(),49 withdrawn: z.boolean(),50 isOver: z.boolean(),51});52export type Lot = z.infer<typeof LotSchema>;5354export const PagePayloadSchema = z.object({55 kind: z.literal('sale_lots'),56 sale: SaleSchema,57 page: z.number().int(),58 totalHits: z.number().nullable(),59 lots: z.array(LotSchema),60});61export type PagePayload = z.infer<typeof PagePayloadSchema>;6263const ConfigSchema = z.object({64 categoryLabels: z.array(z.string()).default([]),65 monthsPerRun: z.number().int().min(1).default(2),66 backfillMonths: z.number().int().min(1).default(36),67 maxSalesPerRun: z.number().int().min(1).default(30),68 pageSize: z.number().int().min(10).max(200).default(84),69});7071function numOrNull(v: unknown): number | null {72 if (v === null || v === undefined || v === '') return null;73 const n = typeof v === 'number' ? v : Number.parseFloat(String(v));74 return Number.isFinite(n) ? n : null;75}76const str = (v: unknown): string | null => (typeof v === 'string' && v.trim().length ? v.trim() : null);7778/** Parse an auctionresults month response into sales; `categoryNames` maps filter ids → labels. */79export function parseResultsMonth(json: any): Sale[] {80 const labels = new Map<string, string>();81 for (const g of json?.filters?.groups ?? []) {82 for (const f of g.filters ?? []) if (f.id && f.label_txt) labels.set(String(f.id), String(f.label_txt));83 for (const fg of Object.values((g.filter_groups ?? {}) as Record<string, any>)) for (const f of fg?.filters ?? []) if (f.id && f.label_txt) labels.set(String(f.id), String(f.label_txt));84 }85 const out: Sale[] = [];86 for (const e of json?.events ?? []) {87 const landing = str(e.landing_url);88 const saleNumber = landing?.match(/SaleNumber=(\d+)/)?.[1] ?? String(e.subtitle_txt ?? '').match(/Auction\s+(\d{4,6})/)?.[1] ?? null;89 if (!e.event_id || !saleNumber) continue;90 const ids = String(e.filter_ids ?? '').split('|').filter(Boolean);91 const categoryLabels = ids.filter((id) => id.startsWith('category_')).map((id) => labels.get(id) ?? id);92 const locId = ids.find((id) => id.startsWith('location_'));93 out.push(94 SaleSchema.parse({95 saleId: String(e.event_id),96 saleNumber,97 title: String(e.title_txt ?? ''),98 subtitle: str(e.subtitle_txt),99 eventType: /online/i.test(String(e.subtitle_txt ?? '')) || ids.includes('event_115') ? 'Online' : ids.includes('event_live') ? 'Live' : null,100 location: str(e.location_txt) ?? (locId ? labels.get(locId) ?? null : null),101 startDate: str(e.start_date),102 endDate: str(e.end_date),103 landingUrl: landing,104 categoryLabels,105 saleTotalText: str(e.sale_total_value_txt),106 }),107 );108 }109 return out;110}111112export function parseLotSearch(json: any): { lots: Lot[]; total: number | null } {113 const lots: Lot[] = [];114 for (const l of json?.lots ?? []) {115 lots.push(116 LotSchema.parse({117 objectId: String(l.object_id ?? ''),118 lotNumber: String(l.lot_id_txt ?? ''),119 titlePrimary: String(l.title_primary_txt ?? '').trim(),120 titleSecondary: str(l.title_secondary_txt),121 titleTertiary: str(l.title_tertiary_txt),122 description: str(String(l.description_txt ?? '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')),123 url: str(l.url),124 imageUrl: str(l.image?.image_src),125 estimateLow: numOrNull(l.estimate_low),126 estimateHigh: numOrNull(l.estimate_high),127 estimateText: str(l.estimate_txt),128 priceRealised: numOrNull(l.price_realised),129 priceRealisedText: str(l.price_realised_txt),130 startDate: str(l.start_date),131 endDate: str(l.end_date),132 withdrawn: Boolean(l.lot_withdrawn),133 isOver: Boolean(l.is_auction_over),134 }),135 );136 }137 return { lots, total: numOrNull(json?.total_hits_filtered) };138}139140export function resultsUrl(month: number, year: number): string {141 return `${RESULTS}?language=en&month=${month}&year=${year}`;142}143export function lotSearchUrl(sale: Pick<Sale, 'saleId' | 'saleNumber'>, page: number, pageSize: number): string {144 return `${LOTSEARCH}?language=en&SaleNumber=${sale.saleNumber}&SaleId=${sale.saleId}&page=${page}&pageSize=${pageSize}&sortby=lotnumber`;145}146147/** "GBP 190,500" → { currency, amount } (amount taken from numeric price_realised when available). */148export function currencyFromText(text: string | null): CurrencyCode | null {149 const code = text?.match(/^([A-Z]{3})\b/)?.[1] ?? null;150 return code && (SUPPORTED_CURRENCIES as readonly string[]).includes(code) ? (code as CurrencyCode) : null;151}152153export function hintForSale(sale: Sale, cfg: string[]): DeptHint {154 const labels = sale.categoryLabels.length ? sale.categoryLabels : [];155 const chosen = labels.find((l) => cfg.includes(l)) ?? labels[0] ?? null;156 const fromLabel = hintFromLabel(chosen);157 if (fromLabel !== 'unknown') return fromLabel;158 if (chosen === 'Collectibles') return 'popular_culture';159 return hintFromLabel(sale.title);160}161162function monthsBack(from: Date, n: number): Array<{ month: number; year: number }> {163 const out: Array<{ month: number; year: number }> = [];164 const d = new Date(Date.UTC(from.getUTCFullYear(), from.getUTCMonth(), 1));165 for (let i = 0; i < n; i++) {166 out.push({ month: d.getUTCMonth() + 1, year: d.getUTCFullYear() });167 d.setUTCMonth(d.getUTCMonth() - 1);168 }169 return out;170}171172export default function createConnector(meta: ConnectorMeta) {173 return new ChristiesConnector(meta);174}175176export class ChristiesConnector extends BaseConnector {177 readonly version = '1.0.0';178 readonly parserVersion = '1.0.0';179 protected override minIntervalMs = 1200;180 private readonly config = ConfigSchema.parse(this.meta.config ?? {});181182 private async json(ctx: CrawlContext, url: string): Promise<unknown | null> {183 await this.throttle();184 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0, headers: { accept: 'application/json', referer: `${SITE}/en/results`, 'user-agent': UA } });185 if (!res.success || res.json === null || res.json === undefined) {186 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);187 return null;188 }189 return res.json;190 }191192 private wanted(sale: Sale): boolean {193 if (!this.config.categoryLabels.length) return true;194 return sale.categoryLabels.some((l) => this.config.categoryLabels.includes(l));195 }196197 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {198 const cursor = { ...(ctx.options.cursor ?? {}) } as { done?: string[]; backfillMonth?: string };199 const done = new Set(cursor.done ?? []);200 const probe = ctx.options.mode === 'probe';201 const backfill = ctx.options.mode === 'backfill';202 const now = new Date();203 let months = monthsBack(now, probe ? 1 : backfill ? this.config.backfillMonths : this.config.monthsPerRun);204 if (backfill && cursor.backfillMonth) {205 const [y, m] = cursor.backfillMonth.split('-').map(Number);206 months = months.filter((x) => x.year < y! || (x.year === y && x.month <= m!));207 }208 let sales = 0;209 let yielded = 0;210 for (const { month, year } of months) {211 const monthJson = await this.json(ctx, resultsUrl(month, year));212 if (!monthJson) continue;213 const list = parseResultsMonth(monthJson).filter((s) => this.wanted(s));214 if (!list.length && !(monthJson as { events?: unknown[] }).events?.length) ctx.anomaly('empty_page', `results ${year}-${month}: no events`);215 for (const sale of list) {216 if (done.has(sale.saleId)) continue;217 if (sales >= (probe ? 1 : this.config.maxSalesPerRun)) return;218 sales++;219 let page = 1;220 let seen = 0;221 for (; page <= 40; page++) {222 const url = lotSearchUrl(sale, page, probe ? 20 : this.config.pageSize);223 const lj = await this.json(ctx, url);224 if (!lj) break;225 const { lots, total } = parseLotSearch(lj);226 if (!lots.length) break;227 seen += lots.length;228 const payload: PagePayload = { kind: 'sale_lots', sale, page, totalHits: total, lots };229 yield { url, externalId: `${sale.saleNumber}#${page}`, kind: 'sale', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };230 yielded += lots.length;231 if (this.reached(ctx, yielded)) return;232 if (probe || (total !== null && seen >= total)) break;233 }234 done.add(sale.saleId);235 cursor.done = [...done].slice(-800);236 await ctx.setCursor(cursor);237 }238 if (backfill) {239 cursor.backfillMonth = `${year}-${String(month).padStart(2, '0')}`;240 await ctx.setCursor(cursor);241 }242 }243 }244245 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {246 const page = PagePayloadSchema.parse(raw.payload);247 const sale = page.sale;248 const hint = hintForSale(sale, this.config.categoryLabels);249 const out: NormalizedRecord[] = [];250 for (const lot of page.lots) {251 if (lot.withdrawn || lot.priceRealised === null || lot.priceRealised <= 0) continue;252 const cur = currencyFromText(lot.priceRealisedText) ?? currencyFromText(lot.estimateText);253 if (!cur) continue;254 const title = [lot.titlePrimary, lot.titleSecondary, lot.titleTertiary].filter(Boolean).join(' — ');255 const categorySlug = slugFromTitle(title, hint) ?? slugFromTitle(lot.description ?? '', hint);256 if (!categorySlug) continue;257 const saleDate = new Date(lot.endDate ?? sale.endDate ?? '');258 if (Number.isNaN(saleDate.getTime()) || saleDate.getTime() > Date.now() + 86_400_000) continue;259 const grade = parseGradeFromTitle(title);260 const isWatch = ['rolex', 'patek_philippe', 'audemars_piguet', 'omega', 'other_watches'].includes(categorySlug);261 const reference = isWatch ? watchReference(`${title} ${lot.description ?? ''}`) : null;262 const identifiers: Record<string, string> = { christies_object_id: lot.objectId, christies_sale_number: sale.saleNumber };263 if (reference) identifiers.reference = reference;264 if (categorySlug === 'lego_sets') {265 const n = legoSetNumber(title);266 if (n) identifiers.lego_set_number = n;267 }268 out.push(269 NormalizedSaleSchema.parse({270 kind: 'sale',271 connectorId: this.meta.id,272 sourceId: this.meta.sourceId,273 sourceUrl: lot.url ?? `${SITE}/en/lot/lot-${lot.objectId}`,274 externalId: lot.objectId,275 rawTitle: title,276 description: lot.description,277 imageUrls: lot.imageUrl ? [lot.imageUrl] : [],278 attributes: {279 categorySlug,280 name: lot.titleSecondary && isWatch ? `${lot.titlePrimary} ${lot.titleSecondary}` : title,281 brand: brandFromSlug(categorySlug, title) ?? (isWatch ? lot.titlePrimary.replace(/\.$/, '') : null),282 reference,283 year: safeYear(title),284 identifiers,285 metadata: { sale_id: sale.saleId, sale_number: sale.saleNumber, sale_title: sale.title, sale_type: sale.eventType, sale_categories: sale.categoryLabels, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, estimate_text: lot.estimateText },286 },287 grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null },288 condition: {},289 observedAt: raw.fetchedAt,290 confidence: 0.9,291 parserVersion: this.parserVersion,292 saleType: 'auction',293 saleDate,294 price: lot.priceRealised,295 currency: cur,296 buyerPremiumIncluded: true,297 quantity: 1,298 isBundle: isBundleTitle(title),299 location: sale.location,300 auctionHouse: "Christie's",301 lotNumber: lot.lotNumber,302 }),303 );304 }305 return out;306 }307}308