spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';45/**6 * Artcurial sale results. One raw record per sale (compact lot list parsed from the rendered7 * markdown); normalise → one sale per sold lot (EUR) and one auction_lot for unsold lots.8 */910const BASE = 'https://www.artcurial.com';11const PARSER_VERSION = '1.0.0';1213export const SaleRefSchema = z.object({ number: z.string(), title: z.string(), subtitle: z.string().nullable(), date: z.string().nullable(), location: z.string().nullable(), url: z.string(), online: z.boolean() });14export type SaleRef = z.infer<typeof SaleRefSchema>;15export const LotSchema = z.object({ lotNo: z.string(), title: z.string(), subtitle: z.string().nullable(), estimateLow: z.number().nullable(), estimateHigh: z.number().nullable(), sold: z.number().nullable(), url: z.string(), image: z.string().nullable() });16export const SalePayloadSchema = z.object({ kind: z.literal('sale_results'), url: z.string(), sale: SaleRefSchema, lotCount: z.number().nullable(), lots: z.array(LotSchema) });17export type SalePayload = z.infer<typeof SalePayloadSchema>;1819function lines(block: string): string[] {20  return block21    .split(/\\\\\n|\n/)22    .map((l) => l.replace(/\\$/g, '').replace(/\\\|/g, '|').trim())23    .filter((l) => l && l !== '\\' && l !== '|' && !l.startsWith('- '));24}2526/** Find markdown link blocks "[...](url)" whose URL matches `urlRe`, handling nested image brackets. */27export function linkBlocks(md: string, urlRe: RegExp): Array<{ text: string; url: string }> {28  const out: Array<{ text: string; url: string }> = [];29  const re = new RegExp(`\\]\\((${urlRe.source})\\)`, 'g');30  let m: RegExpExecArray | null;31  while ((m = re.exec(md))) {32    const end = m.index;33    let depth = 0;34    let start = -1;35    for (let i = end - 1; i >= 0; i--) {36      const ch = md[i];37      if (ch === ']') depth++;38      else if (ch === '[') {39        if (depth === 0) {40          start = i;41          break;42        }43        depth--;44      }45    }46    if (start < 0) continue;47    out.push({ text: md.slice(start + 1, end), url: m[1]! });48  }49  return out;50}5152/** Parse the results index: blocks ending in (https://www.artcurial.com/en/sales/<n>). */53export function parseResultsIndex(md: string): SaleRef[] {54  const out: SaleRef[] = [];55  const seen = new Set<string>();56  for (const b of linkBlocks(md, /https:\/\/www\.artcurial\.com\/en\/sales\/[A-Za-z0-9-]+/)) {57    const number = b.url.replace(/.*\/sales\//, '');58    if (seen.has(number)) continue;59    const ls = lines(b.text).filter((l) => !l.startsWith('!['));60    const dateIdx = ls.findIndex((l) => /^[A-Z][a-z]{2} \d{1,2}, \d{4}$/.test(l));61    if (dateIdx < 0) continue;62    const date = parseSourceDate(ls[dateIdx]!);63    const numIdx = ls.findIndex((l, i) => i > dateIdx && l.replace(/\s/g, '') === number.replace(/\s/g, ''));64    const title = ls[numIdx + 1] ?? ls[dateIdx + 2] ?? number;65    const subtitle = ls[numIdx + 2] && !/^\d{1,2}:\d{2}|Online Only|Sessions/i.test(ls[numIdx + 2]!) ? ls[numIdx + 2]! : null;66    const online = ls.some((l) => /online only/i.test(l));67    const location = ls.find((l) => /Artcurial, |Hôtel|Monte-Carlo|Paris|Marrakech/i.test(l)) ?? (online ? 'Online' : null);68    seen.add(number);69    out.push({ number, title, subtitle, date: date ? date.toISOString() : null, location, url: b.url, online });70  }71  return out;72}7374function eur(s: string | undefined): number | null {75  if (!s) return null;76  const n = Number(s.replace(/[^\d.]/g, ''));77  return Number.isFinite(n) && n > 0 ? n : null;78}7980/** Parse a sale page: lot blocks "[1\\ ![img](u)\\ Title\\ Sub\\ Estimate: €300 - 500\\ Sold€1,589](lot url)". */81export function parseSalePage(md: string, saleNumber: string): { lotCount: number | null; lots: z.infer<typeof LotSchema>[] } {82  const lotCount = Number(md.match(/All Lots \((\d+)\)/)?.[1]) || null;83  const lots: z.infer<typeof LotSchema>[] = [];84  const esc = saleNumber.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');85  for (const b of linkBlocks(md, new RegExp(`https://www\\.artcurial\\.com/en/sales/${esc}/lots/[^)\\s]+`))) {86    const image = b.text.match(/!\[[^\]]*\]\(([^)\s]+)/)?.[1] ?? null;87    const ls = lines(b.text).filter((l) => !l.startsWith('!['));88    const lotNo = ls[0] && /^\d{1,4}[A-Za-z]?$/.test(ls[0]) ? ls[0] : null;89    if (!lotNo) continue;90    const estLine = ls.find((l) => /^Estimate/i.test(l)) ?? '';91    const est = estLine.match(/€\s?([\d,.]+)\s*-\s*€?\s?([\d,.]+)/);92    const soldLine = ls.find((l) => /^Sold/i.test(l));93    const sold = eur(soldLine?.match(/€\s?([\d,.]+)/)?.[1]);94    const body = ls.slice(1).filter((l) => !/^(Estimate|Sold|No reserve)/i.test(l));95    const title = body[0] ?? null;96    if (!title) continue;97    lots.push({ lotNo, title, subtitle: body[1] ?? null, estimateLow: eur(est?.[1]), estimateHigh: eur(est?.[2]), sold, url: b.url, image });98  }99  return { lotCount, lots };100}101102const SALE_CATEGORY: Array<[RegExp, string]> = [103  [/horlogerie|watches|montres/i, 'other_watches'],104  [/hermès|hermes|luxury bags|sacs|handbags|vuitton|chanel/i, 'luxury_handbags'],105  [/joaillerie|bijoux|jewel/i, 'jewelry'],106  [/vins|wine|spiritueux|spirits|whisky/i, 'wine'],107  [/bande dessinée|bandes dessinées|comics|bd\b/i, 'comics'],108  [/photograph/i, 'photography'],109  [/design/i, 'design_furniture'],110  [/livres|books|manuscrits|manuscripts/i, 'books'],111  [/automobiles|motorcars|le mans|voitures|racing|automobilia/i, 'automobiles'],112  [/contemporain|contemporary|urban art|street art|post-war|impressionniste|moderne|modern/i, 'contemporary_art'],113  [/mobilier|furniture|arts décoratifs|decorative|tableaux anciens|old master|asian|asiatique|antiquités|antiquities|orientalist|souvenirs historiques/i, 'antiques'],114];115export function categoryForSale(title: string): string {116  return SALE_CATEGORY.find(([re]) => re.test(title))?.[1] ?? 'art';117}118const LOT_CATEGORY: Array<[RegExp, string]> = [119  [/\b(poster|affiche|programme|photograph|helmet|casque|trophy|trophée|book|livre|drawing|dessin|sign|plaque|model|maquette|miniature)s?\b/i, 'automotive_memorabilia'],120  [/rolex|patek|omega|wristwatch|montre/i, 'other_watches'],121];122export function categoryForLot(base: string, title: string): string {123  if (base !== 'automobiles') return base;124  return LOT_CATEGORY.find(([re]) => re.test(title))?.[1] ?? base;125}126127export class ArtcurialConnector extends BaseConnector {128  readonly version = '1.0.0';129  readonly parserVersion = PARSER_VERSION;130  protected override minIntervalMs = 3000;131132  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {133    const resultsUrl = String(this.meta.config.resultsUrl ?? `${BASE}/en/results-auctions-sales`);134    const perRun = Number(this.meta.config.salesPerRun ?? 3);135    const cursor = (ctx.options.cursor ?? {}) as { done?: string[] };136    const done = new Set<string>(cursor.done ?? []);137    const idx = await ctx.fetch(resultsUrl, { engines: ['firecrawl', 'scrapfly'], waitForMs: 5000, timeoutMs: 90_000, minQuality: 0 });138    if (!idx.success || !idx.markdown) {139      ctx.anomaly('results_index_failed', idx.error ?? String(idx.httpStatus));140      return;141    }142    const sales = parseResultsIndex(idx.markdown).filter((s) => !s.date || new Date(s.date).getTime() <= Date.now());143    sales.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? ''));144    let fetched = 0;145    let count = 0;146    for (const sale of sales) {147      if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break;148      if (done.has(sale.number) && ctx.options.mode !== 'backfill') continue;149      await this.throttle();150      fetched++;151      const res = await ctx.fetch(sale.url, {152        engines: ['firecrawl', 'scrapfly'],153        waitForMs: 5000,154        timeoutMs: 90_000,155        expect: ['title', 'price', 'currency'],156        parse: (r) => {157          const p = r.markdown ? parseSalePage(r.markdown, sale.number) : null;158          const sold = p?.lots.find((l) => l.sold);159          return p?.lots.length ? { title: p.lots[0]!.title, price: sold?.sold ?? null, currency: sold ? 'EUR' : null } : null;160        },161      });162      const parsed = res.success && res.markdown ? parseSalePage(res.markdown, sale.number) : null;163      if (!parsed || parsed.lots.length === 0) {164        ctx.anomaly('sale_parse_failed', `${sale.number}: ${res.error ?? res.httpStatus}`);165        continue;166      }167      if (parsed.lotCount && parsed.lots.length < parsed.lotCount * 0.5) ctx.anomaly('partial_lot_list', `${sale.number}: ${parsed.lots.length}/${parsed.lotCount}`);168      const payload: SalePayload = { kind: 'sale_results', url: sale.url, sale, lotCount: parsed.lotCount, lots: parsed.lots };169      count++;170      done.add(sale.number);171      await ctx.setCursor({ done: [...done].slice(-400) });172      yield { url: sale.url, externalId: sale.number, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };173    }174  }175176  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {177    const p = SalePayloadSchema.parse(raw.payload);178    const saleDate = p.sale.date ? new Date(p.sale.date) : null;179    const base = categoryForSale(`${p.sale.title} ${p.sale.subtitle ?? ''}`);180    const out: NormalizedRecord[] = [];181    for (const lot of p.lots) {182      const text = `${lot.title} ${lot.subtitle ?? ''}`;183      const attributes = AssetAttributesSchema.parse({184        categorySlug: categoryForLot(base, text),185        name: lot.title,186        model: lot.subtitle,187        year: extractYear(text),188        identifiers: { artcurial_lot: `${p.sale.number}/${lot.lotNo}` },189        metadata: { sale_number: p.sale.number, sale_title: p.sale.title, sale_subtitle: p.sale.subtitle, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, online_only: p.sale.online },190      });191      const common = {192        connectorId: this.meta.id,193        sourceId: this.meta.sourceId,194        sourceUrl: lot.url,195        externalId: `${p.sale.number}:${lot.lotNo}`,196        rawTitle: lot.subtitle ? `${lot.title} — ${lot.subtitle}` : lot.title,197        description: null,198        imageUrls: lot.image ? [lot.image] : [],199        attributes,200        grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },201        condition: { condition: null, conditionRaw: null, completeness: null },202        observedAt: raw.fetchedAt,203        confidence: 0.88,204        parserVersion: PARSER_VERSION,205      };206      if (lot.sold && saleDate) {207        out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...common, saleType: 'auction', saleDate, price: lot.sold, currency: 'EUR', buyerPremiumIncluded: null, quantity: 1, isBundle: /\b(lot of|ensemble de|set of|\d+\s+(pieces|pièces))\b/i.test(text), location: p.sale.location, auctionHouse: 'Artcurial', lotNumber: lot.lotNo }));208      } else {209        out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...common, auctionHouse: 'Artcurial', auctionName: p.sale.title, lotNumber: lot.lotNo, startsAt: saleDate, endsAt: saleDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: null, currency: 'EUR', status: saleDate && saleDate.getTime() < Date.now() ? 'ended' : 'unknown', location: p.sale.location }));210      }211    }212    return out;213  }214}215216export default (meta: ConnectorMeta) => new ArtcurialConnector(meta);217