import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, parseSourceDate, type NormalizedRecord } from '@rareindex/shared'; /** * Artcurial sale results. One raw record per sale (compact lot list parsed from the rendered * markdown); normalise → one sale per sold lot (EUR) and one auction_lot for unsold lots. */ const BASE = 'https://www.artcurial.com'; const PARSER_VERSION = '1.0.0'; export const SaleRefSchema = z.object({ number: z.string(), title: z.string(), subtitle: z.string().nullable(), date: z.string().nullable(), location: z.string().nullable(), url: z.string(), online: z.boolean() }); export type SaleRef = z.infer; export const LotSchema = z.object({ lotNo: z.string(), title: z.string(), subtitle: z.string().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), sold: z.number().nullable(), url: z.string(), image: z.string().nullable() }); export const SalePayloadSchema = z.object({ kind: z.literal('sale_results'), url: z.string(), sale: SaleRefSchema, lotCount: z.number().nullable(), lots: z.array(LotSchema) }); export type SalePayload = z.infer; function lines(block: string): string[] { return block .split(/\\\\\n|\n/) .map((l) => l.replace(/\\$/g, '').replace(/\\\|/g, '|').trim()) .filter((l) => l && l !== '\\' && l !== '|' && !l.startsWith('- ')); } /** Find markdown link blocks "[...](url)" whose URL matches `urlRe`, handling nested image brackets. */ export function linkBlocks(md: string, urlRe: RegExp): Array<{ text: string; url: string }> { const out: Array<{ text: string; url: string }> = []; const re = new RegExp(`\\]\\((${urlRe.source})\\)`, 'g'); let m: RegExpExecArray | null; while ((m = re.exec(md))) { const end = m.index; let depth = 0; let start = -1; for (let i = end - 1; i >= 0; i--) { const ch = md[i]; if (ch === ']') depth++; else if (ch === '[') { if (depth === 0) { start = i; break; } depth--; } } if (start < 0) continue; out.push({ text: md.slice(start + 1, end), url: m[1]! }); } return out; } /** Parse the results index: blocks ending in (https://www.artcurial.com/en/sales/). */ export function parseResultsIndex(md: string): SaleRef[] { const out: SaleRef[] = []; const seen = new Set(); for (const b of linkBlocks(md, /https:\/\/www\.artcurial\.com\/en\/sales\/[A-Za-z0-9-]+/)) { const number = b.url.replace(/.*\/sales\//, ''); if (seen.has(number)) continue; const ls = lines(b.text).filter((l) => !l.startsWith('![')); const dateIdx = ls.findIndex((l) => /^[A-Z][a-z]{2} \d{1,2}, \d{4}$/.test(l)); if (dateIdx < 0) continue; const date = parseSourceDate(ls[dateIdx]!); const numIdx = ls.findIndex((l, i) => i > dateIdx && l.replace(/\s/g, '') === number.replace(/\s/g, '')); const title = ls[numIdx + 1] ?? ls[dateIdx + 2] ?? number; const subtitle = ls[numIdx + 2] && !/^\d{1,2}:\d{2}|Online Only|Sessions/i.test(ls[numIdx + 2]!) ? ls[numIdx + 2]! : null; const online = ls.some((l) => /online only/i.test(l)); const location = ls.find((l) => /Artcurial, |Hôtel|Monte-Carlo|Paris|Marrakech/i.test(l)) ?? (online ? 'Online' : null); seen.add(number); out.push({ number, title, subtitle, date: date ? date.toISOString() : null, location, url: b.url, online }); } return out; } function eur(s: string | undefined): number | null { if (!s) return null; const n = Number(s.replace(/[^\d.]/g, '')); return Number.isFinite(n) && n > 0 ? n : null; } /** Parse a sale page: lot blocks "[1\\ ![img](u)\\ Title\\ Sub\\ Estimate: €300 - 500\\ Sold€1,589](lot url)". */ export function parseSalePage(md: string, saleNumber: string): { lotCount: number | null; lots: z.infer[] } { const lotCount = Number(md.match(/All Lots \((\d+)\)/)?.[1]) || null; const lots: z.infer[] = []; const esc = saleNumber.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); for (const b of linkBlocks(md, new RegExp(`https://www\\.artcurial\\.com/en/sales/${esc}/lots/[^)\\s]+`))) { const image = b.text.match(/!\[[^\]]*\]\(([^)\s]+)/)?.[1] ?? null; const ls = lines(b.text).filter((l) => !l.startsWith('![')); const lotNo = ls[0] && /^\d{1,4}[A-Za-z]?$/.test(ls[0]) ? ls[0] : null; if (!lotNo) continue; const estLine = ls.find((l) => /^Estimate/i.test(l)) ?? ''; const est = estLine.match(/€\s?([\d,.]+)\s*-\s*€?\s?([\d,.]+)/); const soldLine = ls.find((l) => /^Sold/i.test(l)); const sold = eur(soldLine?.match(/€\s?([\d,.]+)/)?.[1]); const body = ls.slice(1).filter((l) => !/^(Estimate|Sold|No reserve)/i.test(l)); const title = body[0] ?? null; if (!title) continue; lots.push({ lotNo, title, subtitle: body[1] ?? null, estimateLow: eur(est?.[1]), estimateHigh: eur(est?.[2]), sold, url: b.url, image }); } return { lotCount, lots }; } const SALE_CATEGORY: Array<[RegExp, string]> = [ [/horlogerie|watches|montres/i, 'other_watches'], [/hermès|hermes|luxury bags|sacs|handbags|vuitton|chanel/i, 'luxury_handbags'], [/joaillerie|bijoux|jewel/i, 'jewelry'], [/vins|wine|spiritueux|spirits|whisky/i, 'wine'], [/bande dessinée|bandes dessinées|comics|bd\b/i, 'comics'], [/photograph/i, 'photography'], [/design/i, 'design_furniture'], [/livres|books|manuscrits|manuscripts/i, 'books'], [/automobiles|motorcars|le mans|voitures|racing|automobilia/i, 'automobiles'], [/contemporain|contemporary|urban art|street art|post-war|impressionniste|moderne|modern/i, 'contemporary_art'], [/mobilier|furniture|arts décoratifs|decorative|tableaux anciens|old master|asian|asiatique|antiquités|antiquities|orientalist|souvenirs historiques/i, 'antiques'], ]; export function categoryForSale(title: string): string { return SALE_CATEGORY.find(([re]) => re.test(title))?.[1] ?? 'art'; } const LOT_CATEGORY: Array<[RegExp, string]> = [ [/\b(poster|affiche|programme|photograph|helmet|casque|trophy|trophée|book|livre|drawing|dessin|sign|plaque|model|maquette|miniature)s?\b/i, 'automotive_memorabilia'], [/rolex|patek|omega|wristwatch|montre/i, 'other_watches'], ]; export function categoryForLot(base: string, title: string): string { if (base !== 'automobiles') return base; return LOT_CATEGORY.find(([re]) => re.test(title))?.[1] ?? base; } export class ArtcurialConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; async *crawl(ctx: CrawlContext): AsyncIterable { const resultsUrl = String(this.meta.config.resultsUrl ?? `${BASE}/en/results-auctions-sales`); const perRun = Number(this.meta.config.salesPerRun ?? 3); const cursor = (ctx.options.cursor ?? {}) as { done?: string[] }; const done = new Set(cursor.done ?? []); const idx = await ctx.fetch(resultsUrl, { engines: ['firecrawl', 'scrapfly'], waitForMs: 5000, timeoutMs: 90_000, minQuality: 0 }); if (!idx.success || !idx.markdown) { ctx.anomaly('results_index_failed', idx.error ?? String(idx.httpStatus)); return; } const sales = parseResultsIndex(idx.markdown).filter((s) => !s.date || new Date(s.date).getTime() <= Date.now()); sales.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? '')); let fetched = 0; let count = 0; for (const sale of sales) { if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break; if (done.has(sale.number) && ctx.options.mode !== 'backfill') continue; await this.throttle(); fetched++; const res = await ctx.fetch(sale.url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 5000, timeoutMs: 90_000, expect: ['title', 'price', 'currency'], parse: (r) => { const p = r.markdown ? parseSalePage(r.markdown, sale.number) : null; const sold = p?.lots.find((l) => l.sold); return p?.lots.length ? { title: p.lots[0]!.title, price: sold?.sold ?? null, currency: sold ? 'EUR' : null } : null; }, }); const parsed = res.success && res.markdown ? parseSalePage(res.markdown, sale.number) : null; if (!parsed || parsed.lots.length === 0) { ctx.anomaly('sale_parse_failed', `${sale.number}: ${res.error ?? res.httpStatus}`); continue; } if (parsed.lotCount && parsed.lots.length < parsed.lotCount * 0.5) ctx.anomaly('partial_lot_list', `${sale.number}: ${parsed.lots.length}/${parsed.lotCount}`); const payload: SalePayload = { kind: 'sale_results', url: sale.url, sale, lotCount: parsed.lotCount, lots: parsed.lots }; count++; done.add(sale.number); await ctx.setCursor({ done: [...done].slice(-400) }); yield { url: sale.url, externalId: sale.number, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } async normalize(raw: RawRecordLike): Promise { const p = SalePayloadSchema.parse(raw.payload); const saleDate = p.sale.date ? new Date(p.sale.date) : null; const base = categoryForSale(`${p.sale.title} ${p.sale.subtitle ?? ''}`); const out: NormalizedRecord[] = []; for (const lot of p.lots) { const text = `${lot.title} ${lot.subtitle ?? ''}`; const attributes = AssetAttributesSchema.parse({ categorySlug: categoryForLot(base, text), name: lot.title, model: lot.subtitle, year: extractYear(text), identifiers: { artcurial_lot: `${p.sale.number}/${lot.lotNo}` }, metadata: { sale_number: p.sale.number, sale_title: p.sale.title, sale_subtitle: p.sale.subtitle, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, online_only: p.sale.online }, }); const common = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: lot.url, externalId: `${p.sale.number}:${lot.lotNo}`, rawTitle: lot.subtitle ? `${lot.title} — ${lot.subtitle}` : lot.title, description: null, imageUrls: lot.image ? [lot.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.88, parserVersion: PARSER_VERSION, }; if (lot.sold && saleDate) { out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...common, saleType: 'auction', saleDate, price: lot.sold, currency: 'EUR', buyerPremiumIncluded: null, quantity: 1, isBundle: /\b(lot of|ensemble de|set of|\d+\s+(pieces|pièces))\b/i.test(text), location: p.sale.location, auctionHouse: 'Artcurial', lotNumber: lot.lotNo })); } else { out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...common, auctionHouse: 'Artcurial', auctionName: p.sale.title, lotNumber: lot.lotNo, startsAt: saleDate, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: null, currency: 'EUR', status: saleDate && saleDate.getTime() < Date.now() ? 'ended' : 'unknown', location: p.sale.location })); } } return out; } } export default (meta: ConnectorMeta) => new ArtcurialConnector(meta);