import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { CurrencySchema, parsePrice, type AssetAttributes, type CurrencyCode, type NormalizedAuctionLot, type NormalizedRecord, type NormalizedSale } from '@rareindex/shared'; /** * Phillips watch auction results. One raw record per sale (auction) holding the rendered lot list * markdown; normalisation yields one sale per lot with a realised price (buyer's premium included, * as published by Phillips) plus an auction_lot record for unsold/withdrawn lots. */ const PARSER_VERSION = '1.0.0'; export const SalePayloadSchema = z.object({ kind: z.literal('auction_results'), url: z.string(), saleNumber: z.string(), title: z.string(), location: z.string().nullable(), startDate: z.string().nullable(), endDate: z.string().nullable(), lotCount: z.number().nullable(), markdown: z.string(), }); export type SalePayload = z.infer; export interface PastSale { saleNumber: string; title: string; location: string | null; startDate: string | null; endDate: string | null; } /** Return the JSON object literal enclosing position `pos` (brace matching, string-aware). */ function enclosingObject(text: string, pos: number): string | null { let depth = 0; let j = pos; while (j > 0) { const ch = text[j]; if (ch === '}') depth++; else if (ch === '{') { if (depth === 0) break; depth--; } j--; } if (text[j] !== '{') return null; depth = 0; let inStr = false; for (let k = j; k < text.length; k++) { const ch = text[k]; if (inStr) { if (ch === '\\') k++; else if (ch === '"') inStr = false; continue; } if (ch === '"') inStr = true; else if (ch === '{') depth++; else if (ch === '}') { depth--; if (depth === 0) return text.slice(j, k + 1); } } return null; } /** * Extract past sales from the embedded JSON of /auctions/past. Each sale object carries * saleNumber, auctionTitle, departments[{departmentName}], locationName, start/endDateTimeOffset. * Filter = department name or title contains `filter` (case-insensitive), e.g. "watch". */ export function parsePastSales(htmlText: string, filter: string): PastSale[] { const out = new Map(); const re = /"saleNumber":"([A-Z]{2}\d{6})"/g; let m: RegExpExecArray | null; const f = filter.toLowerCase(); while ((m = re.exec(htmlText))) { const sn = m[1]!; if (out.has(sn)) continue; const literal = enclosingObject(htmlText, m.index); if (!literal) continue; let obj: Record; try { obj = JSON.parse(literal) as Record; } catch { continue; } if (obj.saleNumber !== sn) continue; const title = String(obj.auctionTitle ?? obj.title ?? ''); const departments = Array.isArray(obj.departments) ? (obj.departments as Array<{ departmentName?: string }>).map((d) => String(d.departmentName ?? '')) : []; const hay = `${title} ${departments.join(' ')}`.toLowerCase(); if (f && !hay.includes(f)) continue; out.set(sn, { saleNumber: sn, title, location: typeof obj.locationName === 'string' ? obj.locationName : null, startDate: typeof obj.startDateTimeOffset === 'string' ? obj.startDateTimeOffset : null, endDate: typeof obj.endDateTimeOffset === 'string' ? obj.endDateTimeOffset : null, }); } return [...out.values()]; } export interface ParsedLot { lotNumber: string | null; maker: string; reference: string | null; model: string | null; estimateLow: number | null; estimateHigh: number | null; soldFor: number | null; currency: CurrencyCode | null; url: string; image: string | null; noReserve: boolean; lines: string[]; } const CUR_RE = /^(HK\$|US\$|S\$|CHF|USD|HKD|GBP|EUR|SGD|JPY|\$|£|€)\s?([\d,]+(?:\.\d+)?)/; const SYMBOLS: Record = { 'HK$': 'HKD', 'US$': 'USD', S$: 'SGD', $: 'USD', '£': 'GBP', '€': 'EUR' }; function money(s: string): { amount: number; currency: CurrencyCode | null } | null { const m = s.trim().match(CUR_RE); if (!m) return null; const sym = m[1]!; const currency: CurrencyCode | null = SYMBOLS[sym] ?? (CurrencySchema.safeParse(sym).success ? (sym as CurrencyCode) : null); return { amount: Number(m[2]!.replace(/,/g, '')), currency }; } /** Parse markdown lot cards: "[![img](url)\\ \\ 1\\ \\ Rolex\\ Ref. 116509\\ Cosmograph Daytona\\ \\ Estimate\\ \\ CHF25,000–50,000\\ \\ Sold For\\ \\ CHF48,260](https://www.phillips.com/detail/rolex/214153)" */ export function parseLots(md: string): ParsedLot[] { const out: ParsedLot[] = []; const re = /\[!\[[^\]]*\]\(([^)]+)\)([\s\S]*?)\]\((https:\/\/www\.phillips\.com\/detail\/[^)]+)\)/g; let m: RegExpExecArray | null; while ((m = re.exec(md))) { const image = m[1]!.split(' ')[0] ?? null; const body = m[2]!; const url = m[3]!; const lines = body .split(/\\\\\n|\n/) .map((l) => l.replace(/\\$/g, '').trim()) .filter((l) => l && l !== '\\'); const noReserve = lines.some((l) => /no reserve/i.test(l)); const content = lines.filter((l) => !/no reserve|brought to you/i.test(l)); const lotIdx = content.findIndex((l) => /^\d{1,4}[A-Z]?$/.test(l)); const lotNumber = lotIdx >= 0 ? content[lotIdx]! : null; const estIdx = content.findIndex((l) => /^Estimate$/i.test(l)); const soldIdx = content.findIndex((l) => /^Sold For$/i.test(l)); const descLines = content.slice(lotIdx + 1, estIdx >= 0 ? estIdx : soldIdx >= 0 ? soldIdx : content.length); const maker = descLines[0] ?? ''; const refLine = descLines.find((l) => /^Ref\.?\s/i.test(l)) ?? null; const reference = refLine ? refLine.replace(/^Ref\.?\s*/i, '').trim() : null; const model = descLines.filter((l) => l !== maker && l !== refLine)[0] ?? null; let estimateLow: number | null = null; let estimateHigh: number | null = null; let currency: CurrencyCode | null = null; if (estIdx >= 0 && content[estIdx + 1]) { const est = content[estIdx + 1]!; const parts = est.split(/[–-]/); const lo = money(parts[0]!); if (lo) { estimateLow = lo.amount; currency = lo.currency; const hi = parts[1] ? (money(parts[1]) ?? (Number(parts[1].replace(/[^\d.]/g, '')) || null)) : null; estimateHigh = typeof hi === 'number' ? hi : (hi?.amount ?? null); } } let soldFor: number | null = null; if (soldIdx >= 0 && content[soldIdx + 1]) { const sold = money(content[soldIdx + 1]!); if (sold) { soldFor = sold.amount; currency = sold.currency ?? currency; } } if (!maker) continue; out.push({ lotNumber, maker, reference, model, estimateLow, estimateHigh, soldFor, currency, url, image, noReserve, lines: content }); } return out; } const BRAND_CATEGORY: Array<[RegExp, string]> = [ [/^rolex/i, 'rolex'], [/^patek/i, 'patek_philippe'], [/^audemars/i, 'audemars_piguet'], [/^omega/i, 'omega'], ]; export class PhillipsWatchesConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; async *crawl(ctx: CrawlContext): AsyncIterable { const pastUrl = String(this.meta.config.pastAuctionsUrl ?? 'https://www.phillips.com/auctions/past'); const perRun = Number(this.meta.config.salesPerRun ?? 6); const filter = String(this.meta.config.titleFilter ?? 'watch'); const cursor = ctx.options.cursor ?? {}; const done = new Set((cursor.doneSales as string[] | undefined) ?? []); let sales: PastSale[]; if (ctx.options.seeds?.length) { sales = ctx.options.seeds.map((s) => ({ saleNumber: s.replace(/.*\/auction\//, '').replace(/\/.*$/, ''), title: s, location: null, startDate: null, endDate: null })); } else { const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 }); if (!list.success || !list.html) { ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus)); return; } sales = parsePastSales(list.html, filter).filter((s) => !s.endDate || new Date(s.endDate).getTime() < Date.now()); // newest first sales.sort((a, b) => (b.endDate ?? '').localeCompare(a.endDate ?? '')); } let count = 0; let fetched = 0; for (const sale of sales) { if (ctx.signal?.aborted) return; if (done.has(sale.saleNumber) && ctx.options.mode !== 'backfill') continue; if (this.reached(ctx, count) || fetched >= perRun) break; const url = `https://www.phillips.com/auction/${sale.saleNumber}`; await this.throttle(); fetched++; const res = await ctx.fetch(url, { waitForMs: 6000, timeoutMs: 90_000, expect: ['title', 'price', 'currency'], parse: (r) => { const lots = parseLots(r.markdown ?? ''); const sold = lots.find((l) => l.soldFor); return { title: lots[0]?.maker ?? null, price: sold?.soldFor ?? null, currency: sold?.currency ?? null }; }, }); if (!res.success || !res.markdown) { ctx.anomaly('sale_fetch_failed', `${sale.saleNumber}: ${res.error ?? res.httpStatus}`); continue; } const md = res.markdown; const lotCount = Number(md.match(/##\s+(\d+)\s+Lots/)?.[1]) || null; const title = sale.title || md.match(/^#\s+(.+)$/m)?.[1] || sale.saleNumber; const concluded = md.match(/Concluded\s*([A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})/)?.[1] ?? null; const lotsStart = md.indexOf('## '); const payload: SalePayload = { kind: 'auction_results', url, saleNumber: sale.saleNumber, title, location: sale.location ?? md.match(/\n(Geneva|New York|Hong Kong|London)\n/)?.[1] ?? null, startDate: sale.startDate, endDate: sale.endDate ?? (concluded ? new Date(`${concluded} UTC`).toISOString() : null), lotCount, markdown: md.slice(lotsStart >= 0 ? lotsStart : 0) }; const parsed = parseLots(payload.markdown); if (parsed.length === 0) { ctx.anomaly('parse_failure_lots', sale.saleNumber); continue; } if (lotCount && parsed.length < lotCount * 0.5) ctx.anomaly('partial_lot_list', `${sale.saleNumber}: ${parsed.length}/${lotCount}`); count++; done.add(sale.saleNumber); await ctx.setCursor({ doneSales: [...done].slice(-500), updatedAt: new Date().toISOString() }); yield { url, externalId: sale.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } async normalize(raw: RawRecordLike): Promise { const p = SalePayloadSchema.parse(raw.payload); const lots = parseLots(p.markdown); const saleDate = p.endDate ? new Date(p.endDate) : null; const out: NormalizedRecord[] = []; for (const lot of lots) { const categorySlug = BRAND_CATEGORY.find(([re]) => re.test(lot.maker))?.[1] ?? 'other_watches'; const currency = lot.currency; const attributes: AssetAttributes = { categorySlug, subcategorySlug: null, franchise: null, brand: lot.maker, series: null, set: null, setCode: null, name: `${lot.maker} ${lot.model ?? lot.reference ?? ''}`.trim(), model: lot.model, reference: lot.reference, number: null, year: null, edition: null, variant: null, language: null, region: null, country: null, material: null, size: null, color: null, rarity: null, productionQuantity: null, originalMsrp: null, originalMsrpCurrency: null, identifiers: { ...(lot.reference ? { reference: lot.reference } : {}), phillips_lot: lot.url.replace(/.*\/detail\//, '') }, metadata: { sale_number: p.saleNumber, sale_title: p.title, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, no_reserve: lot.noReserve }, }; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lot.url, externalId: `${p.saleNumber}:${lot.lotNumber ?? lot.url}`, rawTitle: `${lot.maker}${lot.reference ? ` Ref. ${lot.reference}` : ''}${lot.model ? ` ${lot.model}` : ''}`, description: null, imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.92, parserVersion: PARSER_VERSION, }; if (lot.soldFor && currency && saleDate) { const sale: NormalizedSale = { kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.soldFor, currency, buyerPremiumIncluded: true, quantity: 1, isBundle: false, location: p.location, auctionHouse: 'Phillips', lotNumber: lot.lotNumber }; out.push(sale); } else { const lotRec: NormalizedAuctionLot = { kind: 'auction_lot', ...base, auctionHouse: 'Phillips', auctionName: p.title, lotNumber: lot.lotNumber, startsAt: p.startDate ? new Date(p.startDate) : null, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: null, currency, status: saleDate && saleDate.getTime() < Date.now() ? 'ended' : 'unknown', location: p.location }; out.push(lotRec); } } return out; } } export default (meta: ConnectorMeta) => new PhillipsWatchesConnector(meta);