TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { CurrencySchema, parsePrice, type AssetAttributes, type CurrencyCode, type NormalizedAuctionLot, type NormalizedRecord, type NormalizedSale } from '@rareindex/shared';45/**6 * Phillips watch auction results. One raw record per sale (auction) holding the rendered lot list7 * markdown; normalisation yields one sale per lot with a realised price (buyer's premium included,8 * as published by Phillips) plus an auction_lot record for unsold/withdrawn lots.9 */1011const PARSER_VERSION = '1.0.0';1213export const SalePayloadSchema = z.object({14 kind: z.literal('auction_results'),15 url: z.string(),16 saleNumber: z.string(),17 title: z.string(),18 location: z.string().nullable(),19 startDate: z.string().nullable(),20 endDate: z.string().nullable(),21 lotCount: z.number().nullable(),22 markdown: z.string(),23});24export type SalePayload = z.infer<typeof SalePayloadSchema>;2526export interface PastSale {27 saleNumber: string;28 title: string;29 location: string | null;30 startDate: string | null;31 endDate: string | null;32}3334/** Return the JSON object literal enclosing position `pos` (brace matching, string-aware). */35function enclosingObject(text: string, pos: number): string | null {36 let depth = 0;37 let j = pos;38 while (j > 0) {39 const ch = text[j];40 if (ch === '}') depth++;41 else if (ch === '{') {42 if (depth === 0) break;43 depth--;44 }45 j--;46 }47 if (text[j] !== '{') return null;48 depth = 0;49 let inStr = false;50 for (let k = j; k < text.length; k++) {51 const ch = text[k];52 if (inStr) {53 if (ch === '\\') k++;54 else if (ch === '"') inStr = false;55 continue;56 }57 if (ch === '"') inStr = true;58 else if (ch === '{') depth++;59 else if (ch === '}') {60 depth--;61 if (depth === 0) return text.slice(j, k + 1);62 }63 }64 return null;65}6667/**68 * Extract past sales from the embedded JSON of /auctions/past. Each sale object carries69 * saleNumber, auctionTitle, departments[{departmentName}], locationName, start/endDateTimeOffset.70 * Filter = department name or title contains `filter` (case-insensitive), e.g. "watch".71 */72export function parsePastSales(htmlText: string, filter: string): PastSale[] {73 const out = new Map<string, PastSale>();74 const re = /"saleNumber":"([A-Z]{2}\d{6})"/g;75 let m: RegExpExecArray | null;76 const f = filter.toLowerCase();77 while ((m = re.exec(htmlText))) {78 const sn = m[1]!;79 if (out.has(sn)) continue;80 const literal = enclosingObject(htmlText, m.index);81 if (!literal) continue;82 let obj: Record<string, unknown>;83 try {84 obj = JSON.parse(literal) as Record<string, unknown>;85 } catch {86 continue;87 }88 if (obj.saleNumber !== sn) continue;89 const title = String(obj.auctionTitle ?? obj.title ?? '');90 const departments = Array.isArray(obj.departments) ? (obj.departments as Array<{ departmentName?: string }>).map((d) => String(d.departmentName ?? '')) : [];91 const hay = `${title} ${departments.join(' ')}`.toLowerCase();92 if (f && !hay.includes(f)) continue;93 out.set(sn, {94 saleNumber: sn,95 title,96 location: typeof obj.locationName === 'string' ? obj.locationName : null,97 startDate: typeof obj.startDateTimeOffset === 'string' ? obj.startDateTimeOffset : null,98 endDate: typeof obj.endDateTimeOffset === 'string' ? obj.endDateTimeOffset : null,99 });100 }101 return [...out.values()];102}103104export interface ParsedLot {105 lotNumber: string | null;106 maker: string;107 reference: string | null;108 model: string | null;109 estimateLow: number | null;110 estimateHigh: number | null;111 soldFor: number | null;112 currency: CurrencyCode | null;113 url: string;114 image: string | null;115 noReserve: boolean;116 lines: string[];117}118119const CUR_RE = /^(HK\$|US\$|S\$|CHF|USD|HKD|GBP|EUR|SGD|JPY|\$|£|€)\s?([\d,]+(?:\.\d+)?)/;120const SYMBOLS: Record<string, CurrencyCode> = { 'HK$': 'HKD', 'US$': 'USD', S$: 'SGD', $: 'USD', '£': 'GBP', '€': 'EUR' };121function money(s: string): { amount: number; currency: CurrencyCode | null } | null {122 const m = s.trim().match(CUR_RE);123 if (!m) return null;124 const sym = m[1]!;125 const currency: CurrencyCode | null = SYMBOLS[sym] ?? (CurrencySchema.safeParse(sym).success ? (sym as CurrencyCode) : null);126 return { amount: Number(m[2]!.replace(/,/g, '')), currency };127}128129/** Parse markdown lot cards: "[\\ \\ 1\\ \\ Rolex\\ Ref. 116509\\ Cosmograph Daytona\\ \\ Estimate\\ \\ CHF25,000–50,000\\ \\ Sold For\\ \\ CHF48,260](https://www.phillips.com/detail/rolex/214153)" */130export function parseLots(md: string): ParsedLot[] {131 const out: ParsedLot[] = [];132 const re = /\[!\[[^\]]*\]\(([^)]+)\)([\s\S]*?)\]\((https:\/\/www\.phillips\.com\/detail\/[^)]+)\)/g;133 let m: RegExpExecArray | null;134 while ((m = re.exec(md))) {135 const image = m[1]!.split(' ')[0] ?? null;136 const body = m[2]!;137 const url = m[3]!;138 const lines = body139 .split(/\\\\\n|\n/)140 .map((l) => l.replace(/\\$/g, '').trim())141 .filter((l) => l && l !== '\\');142 const noReserve = lines.some((l) => /no reserve/i.test(l));143 const content = lines.filter((l) => !/no reserve|brought to you/i.test(l));144 const lotIdx = content.findIndex((l) => /^\d{1,4}[A-Z]?$/.test(l));145 const lotNumber = lotIdx >= 0 ? content[lotIdx]! : null;146 const estIdx = content.findIndex((l) => /^Estimate$/i.test(l));147 const soldIdx = content.findIndex((l) => /^Sold For$/i.test(l));148 const descLines = content.slice(lotIdx + 1, estIdx >= 0 ? estIdx : soldIdx >= 0 ? soldIdx : content.length);149 const maker = descLines[0] ?? '';150 const refLine = descLines.find((l) => /^Ref\.?\s/i.test(l)) ?? null;151 const reference = refLine ? refLine.replace(/^Ref\.?\s*/i, '').trim() : null;152 const model = descLines.filter((l) => l !== maker && l !== refLine)[0] ?? null;153 let estimateLow: number | null = null;154 let estimateHigh: number | null = null;155 let currency: CurrencyCode | null = null;156 if (estIdx >= 0 && content[estIdx + 1]) {157 const est = content[estIdx + 1]!;158 const parts = est.split(/[–-]/);159 const lo = money(parts[0]!);160 if (lo) {161 estimateLow = lo.amount;162 currency = lo.currency;163 const hi = parts[1] ? (money(parts[1]) ?? (Number(parts[1].replace(/[^\d.]/g, '')) || null)) : null;164 estimateHigh = typeof hi === 'number' ? hi : (hi?.amount ?? null);165 }166 }167 let soldFor: number | null = null;168 if (soldIdx >= 0 && content[soldIdx + 1]) {169 const sold = money(content[soldIdx + 1]!);170 if (sold) {171 soldFor = sold.amount;172 currency = sold.currency ?? currency;173 }174 }175 if (!maker) continue;176 out.push({ lotNumber, maker, reference, model, estimateLow, estimateHigh, soldFor, currency, url, image, noReserve, lines: content });177 }178 return out;179}180181const BRAND_CATEGORY: Array<[RegExp, string]> = [182 [/^rolex/i, 'rolex'],183 [/^patek/i, 'patek_philippe'],184 [/^audemars/i, 'audemars_piguet'],185 [/^omega/i, 'omega'],186];187188export class PhillipsWatchesConnector extends BaseConnector {189 readonly version = '1.0.0';190 readonly parserVersion = PARSER_VERSION;191 protected override minIntervalMs = 3000;192193 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {194 const pastUrl = String(this.meta.config.pastAuctionsUrl ?? 'https://www.phillips.com/auctions/past');195 const perRun = Number(this.meta.config.salesPerRun ?? 6);196 const filter = String(this.meta.config.titleFilter ?? 'watch');197 const cursor = ctx.options.cursor ?? {};198 const done = new Set<string>((cursor.doneSales as string[] | undefined) ?? []);199 let sales: PastSale[];200 if (ctx.options.seeds?.length) {201 sales = ctx.options.seeds.map((s) => ({ saleNumber: s.replace(/.*\/auction\//, '').replace(/\/.*$/, ''), title: s, location: null, startDate: null, endDate: null }));202 } else {203 const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });204 if (!list.success || !list.html) {205 ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus));206 return;207 }208 sales = parsePastSales(list.html, filter).filter((s) => !s.endDate || new Date(s.endDate).getTime() < Date.now());209 // newest first210 sales.sort((a, b) => (b.endDate ?? '').localeCompare(a.endDate ?? ''));211 }212 let count = 0;213 let fetched = 0;214 for (const sale of sales) {215 if (ctx.signal?.aborted) return;216 if (done.has(sale.saleNumber) && ctx.options.mode !== 'backfill') continue;217 if (this.reached(ctx, count) || fetched >= perRun) break;218 const url = `https://www.phillips.com/auction/${sale.saleNumber}`;219 await this.throttle();220 fetched++;221 const res = await ctx.fetch(url, {222 waitForMs: 6000,223 timeoutMs: 90_000,224 expect: ['title', 'price', 'currency'],225 parse: (r) => {226 const lots = parseLots(r.markdown ?? '');227 const sold = lots.find((l) => l.soldFor);228 return { title: lots[0]?.maker ?? null, price: sold?.soldFor ?? null, currency: sold?.currency ?? null };229 },230 });231 if (!res.success || !res.markdown) {232 ctx.anomaly('sale_fetch_failed', `${sale.saleNumber}: ${res.error ?? res.httpStatus}`);233 continue;234 }235 const md = res.markdown;236 const lotCount = Number(md.match(/##\s+(\d+)\s+Lots/)?.[1]) || null;237 const title = sale.title || md.match(/^#\s+(.+)$/m)?.[1] || sale.saleNumber;238 const concluded = md.match(/Concluded\s*([A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})/)?.[1] ?? null;239 const lotsStart = md.indexOf('## ');240 const payload: SalePayload = { kind: 'auction_results', url, saleNumber: sale.saleNumber, title, location: sale.location ?? md.match(/\n(Geneva|New York|Hong Kong|London)\n/)?.[1] ?? null, startDate: sale.startDate, endDate: sale.endDate ?? (concluded ? new Date(`${concluded} UTC`).toISOString() : null), lotCount, markdown: md.slice(lotsStart >= 0 ? lotsStart : 0) };241 const parsed = parseLots(payload.markdown);242 if (parsed.length === 0) {243 ctx.anomaly('parse_failure_lots', sale.saleNumber);244 continue;245 }246 if (lotCount && parsed.length < lotCount * 0.5) ctx.anomaly('partial_lot_list', `${sale.saleNumber}: ${parsed.length}/${lotCount}`);247 count++;248 done.add(sale.saleNumber);249 await ctx.setCursor({ doneSales: [...done].slice(-500), updatedAt: new Date().toISOString() });250 yield { url, externalId: sale.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };251 }252 }253254 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {255 const p = SalePayloadSchema.parse(raw.payload);256 const lots = parseLots(p.markdown);257 const saleDate = p.endDate ? new Date(p.endDate) : null;258 const out: NormalizedRecord[] = [];259 for (const lot of lots) {260 const categorySlug = BRAND_CATEGORY.find(([re]) => re.test(lot.maker))?.[1] ?? 'other_watches';261 const currency = lot.currency;262 const attributes: AssetAttributes = {263 categorySlug,264 subcategorySlug: null,265 franchise: null,266 brand: lot.maker,267 series: null,268 set: null,269 setCode: null,270 name: `${lot.maker} ${lot.model ?? lot.reference ?? ''}`.trim(),271 model: lot.model,272 reference: lot.reference,273 number: null,274 year: null,275 edition: null,276 variant: null,277 language: null,278 region: null,279 country: null,280 material: null,281 size: null,282 color: null,283 rarity: null,284 productionQuantity: null,285 originalMsrp: null,286 originalMsrpCurrency: null,287 identifiers: { ...(lot.reference ? { reference: lot.reference } : {}), phillips_lot: lot.url.replace(/.*\/detail\//, '') },288 metadata: { sale_number: p.saleNumber, sale_title: p.title, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, no_reserve: lot.noReserve },289 };290 const base = {291 connectorId: this.meta.id,292 sourceId: this.meta.sourceId,293 sourceUrl: lot.url,294 externalId: `${p.saleNumber}:${lot.lotNumber ?? lot.url}`,295 rawTitle: `${lot.maker}${lot.reference ? ` Ref. ${lot.reference}` : ''}${lot.model ? ` ${lot.model}` : ''}`,296 description: null,297 imageUrls: lot.image ? [lot.image] : [],298 attributes,299 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },300 condition: { condition: null, conditionRaw: null, completeness: null },301 observedAt: raw.fetchedAt,302 confidence: 0.92,303 parserVersion: PARSER_VERSION,304 };305 if (lot.soldFor && currency && saleDate) {306 const sale: NormalizedSale = { kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.soldFor, currency, buyerPremiumIncluded: true, quantity: 1, isBundle: false, location: p.location, auctionHouse: 'Phillips', lotNumber: lot.lotNumber };307 out.push(sale);308 } else {309 const lotRec: NormalizedAuctionLot = { kind: 'auction_lot', ...base, auctionHouse: 'Phillips', auctionName: p.title, lotNumber: lot.lotNumber, startsAt: p.startDate ? new Date(p.startDate) : null, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: null, currency, status: saleDate && saleDate.getTime() < Date.now() ? 'ended' : 'unknown', location: p.location };310 out.push(lotRec);311 }312 }313 return out;314 }315}316317export default (meta: ConnectorMeta) => new PhillipsWatchesConnector(meta);318