TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';4import { parseLots, parsePastSales, SalePayloadSchema, type PastSale, type SalePayload } from '../phillips-watches/index.js';56/**7 * Phillips art-side departments (Editions, Design, 20th Century & Contemporary Art, Photographs,8 * Jewels, Handbags). Reuses the phillips-watches page parsers; normalisation maps each lot to an9 * art/design/photography/jewelry/handbag sale with artist/maker as `brand`.10 */1112const PARSER_VERSION = '1.0.0';13const DeptSchema = z.object({ filter: z.string(), category: z.string() });1415export const ArtSalePayloadSchema = SalePayloadSchema.extend({ department: DeptSchema });16export type ArtSalePayload = z.infer<typeof ArtSalePayloadSchema>;1718const LOT_CATEGORY: Array<[RegExp, string]> = [19 [/photograph|gelatin silver|c-print|chromogenic|dye transfer|platinum print/i, 'photography'],20 [/wristwatch|rolex|patek|audemars/i, 'other_watches'],21 [/hermès|hermes|chanel|louis vuitton|birkin|kelly bag/i, 'luxury_handbags'],22 [/ring|necklace|bracelet|brooch|earrings|diamond|sapphire|emerald|ruby/i, 'jewelry'],23 [/chair|table|lamp|cabinet|sofa|desk|stool|vase|bench|sideboard|chandelier/i, 'design_furniture'],24];2526export function categoryForLot(base: string, text: string): string {27 return LOT_CATEGORY.find(([re]) => re.test(text))?.[1] ?? base;28}2930export class PhillipsArtConnector extends BaseConnector {31 readonly version = '1.0.0';32 readonly parserVersion = PARSER_VERSION;33 protected override minIntervalMs = 3000;3435 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {36 const pastUrl = String(this.meta.config.pastAuctionsUrl ?? 'https://www.phillips.com/auctions/past');37 const perRun = Number(this.meta.config.salesPerRun ?? 4);38 const departments = z.array(DeptSchema).parse(this.meta.config.departments ?? []);39 const cursor = ctx.options.cursor ?? {};40 const done = new Set<string>((cursor.doneSales as string[] | undefined) ?? []);41 const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });42 if (!list.success || !list.html) {43 ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus));44 return;45 }46 const candidates: Array<{ sale: PastSale; department: z.infer<typeof DeptSchema> }> = [];47 const seen = new Set<string>();48 for (const department of departments) {49 if (ctx.options.categories?.length && !ctx.options.categories.includes(department.category)) continue;50 for (const sale of parsePastSales(list.html, department.filter)) {51 if (seen.has(sale.saleNumber) || /watch/i.test(sale.title)) continue;52 if (sale.endDate && new Date(sale.endDate).getTime() > Date.now()) continue;53 seen.add(sale.saleNumber);54 candidates.push({ sale, department });55 }56 }57 candidates.sort((a, b) => (b.sale.endDate ?? '').localeCompare(a.sale.endDate ?? ''));58 let count = 0;59 let fetched = 0;60 for (const { sale, department } of candidates) {61 if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break;62 if (done.has(sale.saleNumber) && ctx.options.mode !== 'backfill') continue;63 const url = `https://www.phillips.com/auction/${sale.saleNumber}`;64 await this.throttle();65 fetched++;66 const res = await ctx.fetch(url, {67 waitForMs: 6000,68 timeoutMs: 90_000,69 expect: ['title', 'price', 'currency'],70 parse: (r) => {71 const lots = parseLots(r.markdown ?? '');72 const sold = lots.find((l) => l.soldFor);73 return { title: lots[0]?.maker ?? null, price: sold?.soldFor ?? null, currency: sold?.currency ?? null };74 },75 });76 if (!res.success || !res.markdown) {77 ctx.anomaly('sale_fetch_failed', `${sale.saleNumber}: ${res.error ?? res.httpStatus}`);78 continue;79 }80 const md = res.markdown;81 const lotCount = Number(md.match(/##\s+(\d+)\s+Lots/)?.[1]) || null;82 const concluded = md.match(/Concluded\s*([A-Z][a-z]{2}\s+\d{1,2}\s+\d{4})/)?.[1] ?? null;83 const lotsStart = md.indexOf('## ');84 const payload: ArtSalePayload = {85 kind: 'auction_results',86 url,87 saleNumber: sale.saleNumber,88 title: sale.title || md.match(/^#\s+(.+)$/m)?.[1] || sale.saleNumber,89 location: sale.location ?? md.match(/\n(Geneva|New York|Hong Kong|London|Paris)\n/)?.[1] ?? null,90 startDate: sale.startDate,91 endDate: sale.endDate ?? (concluded ? new Date(`${concluded} UTC`).toISOString() : null),92 lotCount,93 markdown: md.slice(lotsStart >= 0 ? lotsStart : 0),94 department,95 };96 const parsed = parseLots(payload.markdown);97 if (parsed.length === 0) {98 ctx.anomaly('parse_failure_lots', sale.saleNumber);99 continue;100 }101 if (lotCount && parsed.length < lotCount * 0.5) ctx.anomaly('partial_lot_list', `${sale.saleNumber}: ${parsed.length}/${lotCount}`);102 count++;103 done.add(sale.saleNumber);104 await ctx.setCursor({ doneSales: [...done].slice(-500), updatedAt: new Date().toISOString() });105 yield { url, externalId: sale.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };106 }107 }108109 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {110 const p = ArtSalePayloadSchema.parse(raw.payload) as ArtSalePayload & SalePayload;111 const lots = parseLots(p.markdown);112 const saleDate = p.endDate ? new Date(p.endDate) : null;113 const out: NormalizedRecord[] = [];114 for (const lot of lots) {115 // For art lots the "maker" line is the artist; "model" holds the work title; "reference" rarely applies.116 const artist = lot.maker;117 const work = lot.model ?? lot.reference ?? null;118 const text = lot.lines.join(' ');119 const attributes = AssetAttributesSchema.parse({120 categorySlug: categoryForLot(p.department.category, text),121 brand: artist,122 name: work ? `${artist} — ${work}` : artist,123 model: work,124 year: extractYear(text.replace(/\b(19|20)\d{2}\s*[–-]\s*(19|20)\d{2}\b/g, '')),125 identifiers: { phillips_lot: lot.url.replace(/.*\/detail\//, '') },126 metadata: { sale_number: p.saleNumber, sale_title: p.title, department: p.department.filter, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, no_reserve: lot.noReserve, lot_lines: lot.lines.slice(0, 6) },127 });128 const base = {129 connectorId: this.meta.id,130 sourceId: this.meta.sourceId,131 sourceUrl: lot.url,132 externalId: `${p.saleNumber}:${lot.lotNumber ?? lot.url}`,133 rawTitle: work ? `${artist} · ${work}` : artist,134 description: null,135 imageUrls: lot.image ? [lot.image] : [],136 attributes,137 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },138 condition: { condition: null, conditionRaw: null, completeness: null },139 observedAt: raw.fetchedAt,140 confidence: 0.9,141 parserVersion: PARSER_VERSION,142 };143 if (lot.soldFor && lot.currency && saleDate) {144 out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.soldFor, currency: lot.currency, buyerPremiumIncluded: true, quantity: 1, isBundle: false, location: p.location, auctionHouse: 'Phillips', lotNumber: lot.lotNumber }));145 } else {146 out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: 'Phillips', auctionName: p.title, lotNumber: lot.lotNumber, startsAt: p.startDate ? new Date(p.startDate) : null, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: null, currency: lot.currency, status: saleDate && saleDate.getTime() < Date.now() ? 'ended' : 'unknown', location: p.location }));147 }148 }149 return out;150 }151}152153export default (meta: ConnectorMeta) => new PhillipsArtConnector(meta);154