spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateWords, lotAttributes, makeSale, money } from '../../firecrawl/_carlib/index.js';5import { safeYear, slugFromTitle } from '../_auction-lib/categories.js';67/**8 * Heffel (Canada) — auction results with prices realized (CAD, buyer's premium included).9 * Public ASP.NET pages, plain HTTPS. See meta.json accessNotes.10 */1112const SITE = 'https://www.heffel.com';13const PARSER_VERSION = '1.0.0';14const MONTHS: Record<string, number> = { january: 0, february: 1, march: 2, april: 3, may: 4, june: 5, july: 6, august: 7, september: 8, october: 9, november: 10, december: 11 };1516export const SaleRefSchema = z.object({ label: z.string(), url: z.string() });17export const LotSchema = z.object({ lotNumber: z.string(), title: z.string(), artist: z.string().nullable(), url: z.string(), image: z.string().nullable(), priceText: z.string() });18export const PayloadSchema = z.object({19  kind: z.literal('results_page'),20  sale: SaleRefSchema,21  saleName: z.string().nullable(),22  saleDateText: z.string().nullable(),23  sessions: z.array(z.string()),24  premiumIncluded: z.boolean(),25  lots: z.array(LotSchema),26});27export type Payload = z.infer<typeof PayloadSchema>;2829function clean(s: string): string {30  return s31    .replace(/&amp;/g, '&')32    .replace(/&#0?39;|&apos;/g, "'")33    .replace(/&quot;/g, '"')34    .replace(/&nbsp;/g, ' ')35    .replace(/<[^>]+>/g, ' ')36    .replace(/\s+/g, ' ')37    .trim();38}3940/** Results index → sale references (label like "Spring 2026" / "August 2026"). */41export function parseResultsIndex(html: string): z.infer<typeof SaleRefSchema>[] {42  const out: z.infer<typeof SaleRefSchema>[] = [];43  const re = /<a[^>]+href="(\/Links\/Results_E(?:\.aspx)?\?Request=[^"]+)"[^>]*>([\s\S]*?)<\/a>/g;44  let m: RegExpExecArray | null;45  while ((m = re.exec(html))) {46    const label = clean(m[2]!);47    const url = `${SITE}${m[1]!.replace(/&amp;/g, '&')}`;48    if (!label || out.some((s) => s.url === url)) continue;49    out.push({ label, url });50  }51  return out;52}5354/** One results page → sale header + sold lots. */55export function parseResultsPage(html: string, sale: z.infer<typeof SaleRefSchema>): Payload {56  const text = clean(html.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/g, ' '));57  const saleName = text.match(/([A-Z][A-Z0-9 &'’,-]{6,80}?)\s+(?:LIVE\s+)?AUCTION RESULTS/)?.[1]?.trim() ?? null;58  const saleDateText = text.match(/Sale date:\s*([A-Za-z]+,?\s*)?([A-Za-z]+ \d{1,2},? \d{4})/)?.[2] ?? null;59  const sessions = [...text.matchAll(/SESSIONS?:\s*([^()]+?)\s*\(/g)].map((x) => x[1]!.trim()).slice(0, 6);60  const premiumIncluded = /Prices include Buyer'?s Premium/i.test(text);61  const lots: z.infer<typeof LotSchema>[] = [];62  const row = /<tr class="height-adj[^"]*">([\s\S]*?)<\/tr>/g;63  let m: RegExpExecArray | null;64  while ((m = row.exec(html))) {65    const r = m[1]!;66    const num = r.match(/class='text ez'>\s*([0-9]+[A-Za-z]?)\s*</)?.[1];67    const img = r.match(/<img src='([^']+)'/)?.[1];68    const link = r.match(/<a href="(\/Auction\/LotDetails_E(?:\.aspx)?\?Request=[^"]+)">([\s\S]*?)<\/a>/);69    const price = r.match(/<span class="text">\s*(\$[\d,]+(?:\.\d{2})?)\s*<\/span>/)?.[1];70    if (!num || !link || !price) continue;71    const [titlePart, artistPart] = link[2]!.split(/<br\s*\/?>/i);72    lots.push({73      lotNumber: num,74      title: clean(titlePart ?? ''),75      artist: artistPart ? clean(artistPart) || null : null,76      url: `${SITE}${link[1]!.replace(/&amp;/g, '&')}`,77      image: img ? `${SITE}${img.replace(/\/{2,}/g, '/').replace(/^\/?/, '/')}` : null,78      priceText: price,79    });80  }81  return { kind: 'results_page', sale, saleName, saleDateText, sessions, premiumIncluded, lots };82}8384/** "Spring 2026" has no day; "August 2026" → first of month (flagged with lower confidence). */85export function fallbackDate(label: string): Date | null {86  const m = label.match(/([A-Za-z]+)\s+(\d{4})/);87  if (!m) return null;88  const mo = MONTHS[m[1]!.toLowerCase()];89  if (mo === undefined) return null;90  return new Date(Date.UTC(Number(m[2]), mo, 1));91}9293export class HeffelConnector extends BaseConnector {94  readonly version = '1.0.0';95  readonly parserVersion = PARSER_VERSION;96  protected override minIntervalMs = 2000;9798  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {99    const salesPerRun = Number(this.meta.config.salesPerRun ?? 3);100    const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneSales) ? (ctx.options.cursor!.doneSales as string[]) : []);101    const indexUrl = `${SITE}/Links/Results_Choose_E.aspx`;102    await this.throttle();103    const idx = await ctx.fetch(indexUrl, { responseType: 'text', expect: ['title'], parse: (r) => (r.html ? { title: parseResultsIndex(r.html)[0]?.label ?? null } : null) });104    if (!idx.success || !idx.html) {105      ctx.anomaly('page_fetch_failed', `${indexUrl}: ${idx.error ?? idx.httpStatus}`);106      return;107    }108    const sales = parseResultsIndex(idx.html).filter((s) => (ctx.options.mode === 'backfill' ? true : !done.has(s.label)));109    let count = 0;110    for (const sale of sales) {111      if (count >= salesPerRun || ctx.signal?.aborted || this.reached(ctx, count)) break;112      await this.throttle();113      const res = await ctx.fetch(sale.url, {114        responseType: 'text',115        expect: ['title', 'price', 'date'],116        parse: (r) => {117          if (!r.html) return null;118          const p = parseResultsPage(r.html, sale);119          return { title: p.lots[0]?.title ?? null, price: p.lots[0] ? money(p.lots[0].priceText, 'CAD')?.amount ?? null : null, date: p.saleDateText ?? fallbackDate(sale.label) };120        },121      });122      if (!res.success || !res.html) {123        ctx.anomaly('page_fetch_failed', `${sale.url}: ${res.error ?? res.httpStatus}`);124        continue;125      }126      const payload = parseResultsPage(res.html, sale);127      if (payload.lots.length === 0) {128        ctx.anomaly('empty_results_page', sale.label);129        continue;130      }131      count++;132      yield { url: sale.url, externalId: `results:${sale.label}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };133      done.add(sale.label);134      await ctx.setCursor({ doneSales: [...done].slice(-100), updatedAt: new Date().toISOString() });135    }136  }137138  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {139    const p = PayloadSchema.parse(raw.payload);140    const exact = dateWords(p.saleDateText);141    const saleDate = exact ?? fallbackDate(p.sale.label);142    if (!saleDate) return [];143    const out: NormalizedSale[] = [];144    for (const lot of p.lots) {145      const m = money(lot.priceText, 'CAD');146      if (!m) continue;147      const categorySlug = slugFromTitle(`${lot.title} ${lot.artist ?? ''}`, /post-war|contemporary/i.test(p.saleName ?? '') ? 'contemporary' : 'art') ?? 'art';148      const attributes = lotAttributes({149        categorySlug,150        name: lot.artist ? `${lot.artist} — ${lot.title}` : lot.title,151        brand: lot.artist,152        year: safeYear(lot.title),153        country: 'CA',154        identifiers: { heffel_lot: `${p.sale.label.replace(/\s+/g, '-').toLowerCase()}-${lot.lotNumber}` },155        metadata: { sale: p.sale.label, sale_name: p.saleName, sessions: p.sessions, artist: lot.artist },156      });157      out.push(158        makeSale({159          meta: this.meta,160          sourceUrl: lot.url,161          externalId: `${p.sale.label}-${lot.lotNumber}`,162          rawTitle: lot.artist ? `${lot.artist}: ${lot.title}` : lot.title,163          attributes,164          price: m.amount,165          currency: 'CAD',166          saleDate,167          buyerPremiumIncluded: p.premiumIncluded ? true : null,168          auctionHouse: 'Heffel',169          lotNumber: lot.lotNumber,170          imageUrls: lot.image ? [lot.image] : [],171          observedAt: raw.fetchedAt,172          confidence: exact ? 0.9 : 0.7,173          parserVersion: PARSER_VERSION,174          location: 'CA',175        }),176      );177    }178    return out;179  }180}181182export default (meta: ConnectorMeta) => new HeffelConnector(meta);183