import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedSaleSchema, SUPPORTED_CURRENCIES, type CurrencyCode, type NormalizedRecord } from '@rareindex/shared'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import { brandFromSlug, hintFromLabel, isBundleTitle, legoSetNumber, safeYear, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js'; /** * Christie's — prices realised from the public discovery-website JSON endpoints (§110). * Engine: plain HTTPS. See meta.json accessNotes. */ const SITE = 'https://www.christies.com'; const RESULTS = `${SITE}/api/discoverywebsite/auctioncalendar/auctionresults`; const LOTSEARCH = `${SITE}/api/discoverywebsite/auctionpages/lotsearch`; /** christies.com's edge silently drops requests whose User-Agent contains a URL or '@'; we still identify ourselves honestly. */ const UA = 'RareIndex/0.1 (market data research; contact data at rareindex.io)'; export const SaleSchema = z.object({ saleId: z.string(), saleNumber: z.string(), title: z.string(), subtitle: z.string().nullable(), eventType: z.string().nullable(), // Live | Online location: z.string().nullable(), startDate: z.string().nullable(), endDate: z.string().nullable(), landingUrl: z.string().nullable(), categoryLabels: z.array(z.string()), saleTotalText: z.string().nullable(), }); export type Sale = z.infer; export const LotSchema = z.object({ objectId: z.string(), lotNumber: z.string(), titlePrimary: z.string(), titleSecondary: z.string().nullable(), titleTertiary: z.string().nullable(), description: z.string().nullable(), url: z.string().nullable(), imageUrl: z.string().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), estimateText: z.string().nullable(), priceRealised: z.number().nullable(), priceRealisedText: z.string().nullable(), startDate: z.string().nullable(), endDate: z.string().nullable(), withdrawn: z.boolean(), isOver: z.boolean(), }); export type Lot = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('sale_lots'), sale: SaleSchema, page: z.number().int(), totalHits: z.number().nullable(), lots: z.array(LotSchema), }); export type PagePayload = z.infer; const ConfigSchema = z.object({ categoryLabels: z.array(z.string()).default([]), monthsPerRun: z.number().int().min(1).default(2), backfillMonths: z.number().int().min(1).default(36), maxSalesPerRun: z.number().int().min(1).default(30), pageSize: z.number().int().min(10).max(200).default(84), }); function numOrNull(v: unknown): number | null { if (v === null || v === undefined || v === '') return null; const n = typeof v === 'number' ? v : Number.parseFloat(String(v)); return Number.isFinite(n) ? n : null; } const str = (v: unknown): string | null => (typeof v === 'string' && v.trim().length ? v.trim() : null); /** Parse an auctionresults month response into sales; `categoryNames` maps filter ids → labels. */ export function parseResultsMonth(json: any): Sale[] { const labels = new Map(); for (const g of json?.filters?.groups ?? []) { for (const f of g.filters ?? []) if (f.id && f.label_txt) labels.set(String(f.id), String(f.label_txt)); for (const fg of Object.values((g.filter_groups ?? {}) as Record)) for (const f of fg?.filters ?? []) if (f.id && f.label_txt) labels.set(String(f.id), String(f.label_txt)); } const out: Sale[] = []; for (const e of json?.events ?? []) { const landing = str(e.landing_url); const saleNumber = landing?.match(/SaleNumber=(\d+)/)?.[1] ?? String(e.subtitle_txt ?? '').match(/Auction\s+(\d{4,6})/)?.[1] ?? null; if (!e.event_id || !saleNumber) continue; const ids = String(e.filter_ids ?? '').split('|').filter(Boolean); const categoryLabels = ids.filter((id) => id.startsWith('category_')).map((id) => labels.get(id) ?? id); const locId = ids.find((id) => id.startsWith('location_')); out.push( SaleSchema.parse({ saleId: String(e.event_id), saleNumber, title: String(e.title_txt ?? ''), subtitle: str(e.subtitle_txt), eventType: /online/i.test(String(e.subtitle_txt ?? '')) || ids.includes('event_115') ? 'Online' : ids.includes('event_live') ? 'Live' : null, location: str(e.location_txt) ?? (locId ? labels.get(locId) ?? null : null), startDate: str(e.start_date), endDate: str(e.end_date), landingUrl: landing, categoryLabels, saleTotalText: str(e.sale_total_value_txt), }), ); } return out; } export function parseLotSearch(json: any): { lots: Lot[]; total: number | null } { const lots: Lot[] = []; for (const l of json?.lots ?? []) { lots.push( LotSchema.parse({ objectId: String(l.object_id ?? ''), lotNumber: String(l.lot_id_txt ?? ''), titlePrimary: String(l.title_primary_txt ?? '').trim(), titleSecondary: str(l.title_secondary_txt), titleTertiary: str(l.title_tertiary_txt), description: str(String(l.description_txt ?? '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')), url: str(l.url), imageUrl: str(l.image?.image_src), estimateLow: numOrNull(l.estimate_low), estimateHigh: numOrNull(l.estimate_high), estimateText: str(l.estimate_txt), priceRealised: numOrNull(l.price_realised), priceRealisedText: str(l.price_realised_txt), startDate: str(l.start_date), endDate: str(l.end_date), withdrawn: Boolean(l.lot_withdrawn), isOver: Boolean(l.is_auction_over), }), ); } return { lots, total: numOrNull(json?.total_hits_filtered) }; } export function resultsUrl(month: number, year: number): string { return `${RESULTS}?language=en&month=${month}&year=${year}`; } export function lotSearchUrl(sale: Pick, page: number, pageSize: number): string { return `${LOTSEARCH}?language=en&SaleNumber=${sale.saleNumber}&SaleId=${sale.saleId}&page=${page}&pageSize=${pageSize}&sortby=lotnumber`; } /** "GBP 190,500" → { currency, amount } (amount taken from numeric price_realised when available). */ export function currencyFromText(text: string | null): CurrencyCode | null { const code = text?.match(/^([A-Z]{3})\b/)?.[1] ?? null; return code && (SUPPORTED_CURRENCIES as readonly string[]).includes(code) ? (code as CurrencyCode) : null; } export function hintForSale(sale: Sale, cfg: string[]): DeptHint { const labels = sale.categoryLabels.length ? sale.categoryLabels : []; const chosen = labels.find((l) => cfg.includes(l)) ?? labels[0] ?? null; const fromLabel = hintFromLabel(chosen); if (fromLabel !== 'unknown') return fromLabel; if (chosen === 'Collectibles') return 'popular_culture'; return hintFromLabel(sale.title); } function monthsBack(from: Date, n: number): Array<{ month: number; year: number }> { const out: Array<{ month: number; year: number }> = []; const d = new Date(Date.UTC(from.getUTCFullYear(), from.getUTCMonth(), 1)); for (let i = 0; i < n; i++) { out.push({ month: d.getUTCMonth() + 1, year: d.getUTCFullYear() }); d.setUTCMonth(d.getUTCMonth() - 1); } return out; } export default function createConnector(meta: ConnectorMeta) { return new ChristiesConnector(meta); } export class ChristiesConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = '1.0.0'; protected override minIntervalMs = 1200; private readonly config = ConfigSchema.parse(this.meta.config ?? {}); private async json(ctx: CrawlContext, url: string): Promise { await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0, headers: { accept: 'application/json', referer: `${SITE}/en/results`, 'user-agent': UA } }); if (!res.success || res.json === null || res.json === undefined) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return null; } return res.json; } private wanted(sale: Sale): boolean { if (!this.config.categoryLabels.length) return true; return sale.categoryLabels.some((l) => this.config.categoryLabels.includes(l)); } async *crawl(ctx: CrawlContext): AsyncIterable { const cursor = { ...(ctx.options.cursor ?? {}) } as { done?: string[]; backfillMonth?: string }; const done = new Set(cursor.done ?? []); const probe = ctx.options.mode === 'probe'; const backfill = ctx.options.mode === 'backfill'; const now = new Date(); let months = monthsBack(now, probe ? 1 : backfill ? this.config.backfillMonths : this.config.monthsPerRun); if (backfill && cursor.backfillMonth) { const [y, m] = cursor.backfillMonth.split('-').map(Number); months = months.filter((x) => x.year < y! || (x.year === y && x.month <= m!)); } let sales = 0; let yielded = 0; for (const { month, year } of months) { const monthJson = await this.json(ctx, resultsUrl(month, year)); if (!monthJson) continue; const list = parseResultsMonth(monthJson).filter((s) => this.wanted(s)); if (!list.length && !(monthJson as { events?: unknown[] }).events?.length) ctx.anomaly('empty_page', `results ${year}-${month}: no events`); for (const sale of list) { if (done.has(sale.saleId)) continue; if (sales >= (probe ? 1 : this.config.maxSalesPerRun)) return; sales++; let page = 1; let seen = 0; for (; page <= 40; page++) { const url = lotSearchUrl(sale, page, probe ? 20 : this.config.pageSize); const lj = await this.json(ctx, url); if (!lj) break; const { lots, total } = parseLotSearch(lj); if (!lots.length) break; seen += lots.length; const payload: PagePayload = { kind: 'sale_lots', sale, page, totalHits: total, lots }; yield { url, externalId: `${sale.saleNumber}#${page}`, kind: 'sale', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() }; yielded += lots.length; if (this.reached(ctx, yielded)) return; if (probe || (total !== null && seen >= total)) break; } done.add(sale.saleId); cursor.done = [...done].slice(-800); await ctx.setCursor(cursor); } if (backfill) { cursor.backfillMonth = `${year}-${String(month).padStart(2, '0')}`; await ctx.setCursor(cursor); } } } async normalize(raw: RawRecordLike): Promise { const page = PagePayloadSchema.parse(raw.payload); const sale = page.sale; const hint = hintForSale(sale, this.config.categoryLabels); const out: NormalizedRecord[] = []; for (const lot of page.lots) { if (lot.withdrawn || lot.priceRealised === null || lot.priceRealised <= 0) continue; const cur = currencyFromText(lot.priceRealisedText) ?? currencyFromText(lot.estimateText); if (!cur) continue; const title = [lot.titlePrimary, lot.titleSecondary, lot.titleTertiary].filter(Boolean).join(' — '); const categorySlug = slugFromTitle(title, hint) ?? slugFromTitle(lot.description ?? '', hint); if (!categorySlug) continue; const saleDate = new Date(lot.endDate ?? sale.endDate ?? ''); if (Number.isNaN(saleDate.getTime()) || saleDate.getTime() > Date.now() + 86_400_000) continue; const grade = parseGradeFromTitle(title); const isWatch = ['rolex', 'patek_philippe', 'audemars_piguet', 'omega', 'other_watches'].includes(categorySlug); const reference = isWatch ? watchReference(`${title} ${lot.description ?? ''}`) : null; const identifiers: Record = { christies_object_id: lot.objectId, christies_sale_number: sale.saleNumber }; if (reference) identifiers.reference = reference; if (categorySlug === 'lego_sets') { const n = legoSetNumber(title); if (n) identifiers.lego_set_number = n; } out.push( NormalizedSaleSchema.parse({ kind: 'sale', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lot.url ?? `${SITE}/en/lot/lot-${lot.objectId}`, externalId: lot.objectId, rawTitle: title, description: lot.description, imageUrls: lot.imageUrl ? [lot.imageUrl] : [], attributes: { categorySlug, name: lot.titleSecondary && isWatch ? `${lot.titlePrimary} ${lot.titleSecondary}` : title, brand: brandFromSlug(categorySlug, title) ?? (isWatch ? lot.titlePrimary.replace(/\.$/, '') : null), reference, year: safeYear(title), identifiers, metadata: { sale_id: sale.saleId, sale_number: sale.saleNumber, sale_title: sale.title, sale_type: sale.eventType, sale_categories: sale.categoryLabels, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, estimate_text: lot.estimateText }, }, grade: { grader: grade.grader, grade: grade.grade, qualifier: grade.qualifier, certificationNumber: null }, condition: {}, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: this.parserVersion, saleType: 'auction', saleDate, price: lot.priceRealised, currency: cur, buyerPremiumIncluded: true, quantity: 1, isBundle: isBundleTitle(title), location: sale.location, auctionHouse: "Christie's", lotNumber: lot.lotNumber, }), ); } return out; } }