import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; import { dateWords, lotAttributes, makeSale, money } from '../../firecrawl/_carlib/index.js'; import { safeYear, slugFromTitle } from '../_auction-lib/categories.js'; /** * Heffel (Canada) — auction results with prices realized (CAD, buyer's premium included). * Public ASP.NET pages, plain HTTPS. See meta.json accessNotes. */ const SITE = 'https://www.heffel.com'; const PARSER_VERSION = '1.0.0'; const MONTHS: Record = { january: 0, february: 1, march: 2, april: 3, may: 4, june: 5, july: 6, august: 7, september: 8, october: 9, november: 10, december: 11 }; export const SaleRefSchema = z.object({ label: z.string(), url: z.string() }); export const LotSchema = z.object({ lotNumber: z.string(), title: z.string(), artist: z.string().nullable(), url: z.string(), image: z.string().nullable(), priceText: z.string() }); export const PayloadSchema = z.object({ kind: z.literal('results_page'), sale: SaleRefSchema, saleName: z.string().nullable(), saleDateText: z.string().nullable(), sessions: z.array(z.string()), premiumIncluded: z.boolean(), lots: z.array(LotSchema), }); export type Payload = z.infer; function clean(s: string): string { return s .replace(/&/g, '&') .replace(/�?39;|'/g, "'") .replace(/"/g, '"') .replace(/ /g, ' ') .replace(/<[^>]+>/g, ' ') .replace(/\s+/g, ' ') .trim(); } /** Results index → sale references (label like "Spring 2026" / "August 2026"). */ export function parseResultsIndex(html: string): z.infer[] { const out: z.infer[] = []; const re = /]+href="(\/Links\/Results_E(?:\.aspx)?\?Request=[^"]+)"[^>]*>([\s\S]*?)<\/a>/g; let m: RegExpExecArray | null; while ((m = re.exec(html))) { const label = clean(m[2]!); const url = `${SITE}${m[1]!.replace(/&/g, '&')}`; if (!label || out.some((s) => s.url === url)) continue; out.push({ label, url }); } return out; } /** One results page → sale header + sold lots. */ export function parseResultsPage(html: string, sale: z.infer): Payload { const text = clean(html.replace(/|/g, ' ')); const saleName = text.match(/([A-Z][A-Z0-9 &'’,-]{6,80}?)\s+(?:LIVE\s+)?AUCTION RESULTS/)?.[1]?.trim() ?? null; const saleDateText = text.match(/Sale date:\s*([A-Za-z]+,?\s*)?([A-Za-z]+ \d{1,2},? \d{4})/)?.[2] ?? null; const sessions = [...text.matchAll(/SESSIONS?:\s*([^()]+?)\s*\(/g)].map((x) => x[1]!.trim()).slice(0, 6); const premiumIncluded = /Prices include Buyer'?s Premium/i.test(text); const lots: z.infer[] = []; const row = /([\s\S]*?)<\/tr>/g; let m: RegExpExecArray | null; while ((m = row.exec(html))) { const r = m[1]!; const num = r.match(/class='text ez'>\s*([0-9]+[A-Za-z]?)\s*([\s\S]*?)<\/a>/); const price = r.match(/\s*(\$[\d,]+(?:\.\d{2})?)\s*<\/span>/)?.[1]; if (!num || !link || !price) continue; const [titlePart, artistPart] = link[2]!.split(//i); lots.push({ lotNumber: num, title: clean(titlePart ?? ''), artist: artistPart ? clean(artistPart) || null : null, url: `${SITE}${link[1]!.replace(/&/g, '&')}`, image: img ? `${SITE}${img.replace(/\/{2,}/g, '/').replace(/^\/?/, '/')}` : null, priceText: price, }); } return { kind: 'results_page', sale, saleName, saleDateText, sessions, premiumIncluded, lots }; } /** "Spring 2026" has no day; "August 2026" → first of month (flagged with lower confidence). */ export function fallbackDate(label: string): Date | null { const m = label.match(/([A-Za-z]+)\s+(\d{4})/); if (!m) return null; const mo = MONTHS[m[1]!.toLowerCase()]; if (mo === undefined) return null; return new Date(Date.UTC(Number(m[2]), mo, 1)); } export class HeffelConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const salesPerRun = Number(this.meta.config.salesPerRun ?? 3); const done = new Set(Array.isArray(ctx.options.cursor?.doneSales) ? (ctx.options.cursor!.doneSales as string[]) : []); const indexUrl = `${SITE}/Links/Results_Choose_E.aspx`; await this.throttle(); const idx = await ctx.fetch(indexUrl, { responseType: 'text', expect: ['title'], parse: (r) => (r.html ? { title: parseResultsIndex(r.html)[0]?.label ?? null } : null) }); if (!idx.success || !idx.html) { ctx.anomaly('page_fetch_failed', `${indexUrl}: ${idx.error ?? idx.httpStatus}`); return; } const sales = parseResultsIndex(idx.html).filter((s) => (ctx.options.mode === 'backfill' ? true : !done.has(s.label))); let count = 0; for (const sale of sales) { if (count >= salesPerRun || ctx.signal?.aborted || this.reached(ctx, count)) break; await this.throttle(); const res = await ctx.fetch(sale.url, { responseType: 'text', expect: ['title', 'price', 'date'], parse: (r) => { if (!r.html) return null; const p = parseResultsPage(r.html, sale); return { title: p.lots[0]?.title ?? null, price: p.lots[0] ? money(p.lots[0].priceText, 'CAD')?.amount ?? null : null, date: p.saleDateText ?? fallbackDate(sale.label) }; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${sale.url}: ${res.error ?? res.httpStatus}`); continue; } const payload = parseResultsPage(res.html, sale); if (payload.lots.length === 0) { ctx.anomaly('empty_results_page', sale.label); continue; } count++; yield { url: sale.url, externalId: `results:${sale.label}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; done.add(sale.label); await ctx.setCursor({ doneSales: [...done].slice(-100), updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const exact = dateWords(p.saleDateText); const saleDate = exact ?? fallbackDate(p.sale.label); if (!saleDate) return []; const out: NormalizedSale[] = []; for (const lot of p.lots) { const m = money(lot.priceText, 'CAD'); if (!m) continue; const categorySlug = slugFromTitle(`${lot.title} ${lot.artist ?? ''}`, /post-war|contemporary/i.test(p.saleName ?? '') ? 'contemporary' : 'art') ?? 'art'; const attributes = lotAttributes({ categorySlug, name: lot.artist ? `${lot.artist} — ${lot.title}` : lot.title, brand: lot.artist, year: safeYear(lot.title), country: 'CA', identifiers: { heffel_lot: `${p.sale.label.replace(/\s+/g, '-').toLowerCase()}-${lot.lotNumber}` }, metadata: { sale: p.sale.label, sale_name: p.saleName, sessions: p.sessions, artist: lot.artist }, }); out.push( makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.sale.label}-${lot.lotNumber}`, rawTitle: lot.artist ? `${lot.artist}: ${lot.title}` : lot.title, attributes, price: m.amount, currency: 'CAD', saleDate, buyerPremiumIncluded: p.premiumIncluded ? true : null, auctionHouse: 'Heffel', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, confidence: exact ? 0.9 : 0.7, parserVersion: PARSER_VERSION, location: 'CA', }), ); } return out; } } export default (meta: ConnectorMeta) => new HeffelConnector(meta);