TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Generic "past sales → lot results" connector skeleton shared by the g8 auction-house connectors.3 * A house exposes (1) an index of past sales and (2) per-sale lot pages (HTML or embedded JSON) that list4 * lot number, title, realised price and estimate. Subclasses implement the three parsers; crawl5 * (resumable, backfill-aware) and normalize (sale / auction_lot, native currency, premium basis labelled)6 * are shared so every house behaves identically.7 */8import { z } from 'zod';9import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';10import { parseGradeFromTitle } from '@rareindex/taxonomy';11import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, type CurrencyCode, type ExtractionResult, type NormalizedRecord } from '@rareindex/shared';12import { brandFromSlug, hintFromLabel, slugFromTitle, watchReference, type DeptHint } from '../_auction-lib/categories.js';13import { englishLabel, isBundleMultilingual, strongLotHint, yearFromTitle } from './index.js';1415export const SaleRefSchema = z.object({16 id: z.string(),17 title: z.string(),18 url: z.string(),19 /** ISO date (source's own sale date), null when the index does not show it */20 date: z.string().nullable(),21 location: z.string().nullable(),22 extra: z.record(z.string(), z.unknown()).default({}),23});24export type SaleRef = z.infer<typeof SaleRefSchema>;2526export const ParsedLotSchema = z.object({27 lotNo: z.string(),28 title: z.string(),29 subtitle: z.string().nullable().default(null),30 description: z.string().nullable().default(null),31 url: z.string(),32 image: z.string().nullable().default(null),33 /** realised price as published (hammer or premium-inclusive per `premiumIncluded`) */34 price: z.number().nullable().default(null),35 currency: z.string().nullable().default(null),36 premiumIncluded: z.boolean().nullable().default(null),37 estimateLow: z.number().nullable().default(null),38 estimateHigh: z.number().nullable().default(null),39 /** sale date override (timed sales where each lot closes on its own day) */40 date: z.string().nullable().default(null),41 sold: z.boolean(),42 extra: z.record(z.string(), z.unknown()).default({}),43});44export type ParsedLot = z.infer<typeof ParsedLotSchema>;4546export const SaleResultsPayloadSchema = z.object({47 kind: z.literal('sale_results'),48 url: z.string(),49 sale: SaleRefSchema,50 page: z.number(),51 totalLots: z.number().nullable(),52 lots: z.array(ParsedLotSchema),53});54export type SaleResultsPayload = z.infer<typeof SaleResultsPayloadSchema>;5556export interface ParsedSalePage {57 lots: ParsedLot[];58 hasMore: boolean;59 totalLots: number | null;60 /** sale-level facts discovered on the lot page (date, location, premium basis) */61 sale?: Partial<Pick<SaleRef, 'date' | 'location' | 'title'>> & { extra?: Record<string, unknown> };62}6364export interface HouseConfig {65 houseName: string;66 defaultCurrency: CurrencyCode;67 /** default `location` for sale records when the source does not give one */68 location: string | null;69 /** identifiers key, e.g. "aguttes_lot" → "<saleId>/<lotNo>" */70 idKey: string;71 /** default buyer-premium basis when the page does not label it (null = unknown) */72 premiumIncluded: boolean | null;73 /** engines for page fetches (default ['api']) */74 engines?: Array<'api' | 'firecrawl' | 'scrapfly'>;75 responseType?: 'text' | 'json';76 /** taxonomy slug when nothing matches (null = drop the lot) */77 fallbackSlug: string | null;78 /** politeness between requests (ms) */79 minIntervalMs?: number;80 /** cap of sale pages fetched per incremental run */81 maxPagesPerSale?: number;82 /** drop lots whose currency could not be read from the source (multi-currency houses) instead of defaulting */83 requireCurrency?: boolean;84}8586type Cursor = { done?: string[]; pending?: Record<string, string>; backfill?: { index: number; page: number; itemsProcessed: number }; finished?: boolean };8788/** Keep the pending map bounded (most recent 200 entries). */89function trimPending(p: Record<string, string>): Record<string, string> {90 const entries = Object.entries(p).sort((a, b) => b[1].localeCompare(a[1])).slice(0, 200);91 return Object.fromEntries(entries);92}9394export abstract class SaleResultsConnector extends BaseConnector {95 readonly parserVersion = '1.0.0';96 abstract readonly house: HouseConfig;9798 /** Fetch + parse the index of past sales (most recent first is not required; we sort by date). */99 abstract listSales(ctx: CrawlContext): Promise<SaleRef[]>;100 /** URL of page `page` (1-based) of a sale's lot list. */101 abstract salePageUrl(sale: SaleRef, page: number): string;102 /** Parse one lot page (HTML text or JSON). Return null when the document is not a lot page. */103 abstract parseSalePage(res: ExtractionResult, sale: SaleRef, page: number): ParsedSalePage | null;104 /**105 * Taxonomy slug for a lot: strong multilingual lot cues (a Rolex in a "Moderne Kunst" sale is a watch) win,106 * then the department hint derived from the sale title / department (any supported language), then a keyword107 * sweep over title + description, then the house fallback (null = drop).108 */109 categoryFor(sale: SaleRef, lot: ParsedLot): string | null {110 return resolveCategory(`${sale.title} ${String(sale.extra.department ?? '')} ${String(sale.extra.title_fr ?? '')}`, lot, this.house.fallbackSlug);111 }112113 protected get salesPerRun(): number {114 return Number(this.meta.config.salesPerRun ?? 3);115 }116117 protected async fetchSalePage(ctx: CrawlContext, sale: SaleRef, page: number): Promise<{ url: string; res: ExtractionResult; parsed: ParsedSalePage | null }> {118 const url = this.salePageUrl(sale, page);119 await this.throttle(url);120 const res = await ctx.fetch(url, {121 engines: this.house.engines ?? ['api'],122 responseType: this.house.responseType ?? 'text',123 timeoutMs: 60_000,124 expect: ['title', 'price'],125 minQuality: 0.3,126 parse: (r) => {127 const p = this.parseSalePage(r, sale, page);128 const sold = p?.lots.find((l) => l.price);129 return p?.lots.length ? { title: p.lots[0]!.title, price: sold?.price ?? null, currency: sold?.currency ?? null } : null;130 },131 });132 const parsed = res.success ? this.parseSalePage(res, sale, page) : null;133 return { url, res, parsed };134 }135136 /** Yield every page of one sale (stops at hasMore=false, empty page or the per-sale cap). */137 protected async *crawlSale(ctx: CrawlContext, sale: SaleRef, startPage = 1, onPage?: (page: number, lots: number) => Promise<void>): AsyncIterable<RawRecordInput> {138 const cap = this.house.maxPagesPerSale ?? 40;139 let page = startPage;140 for (; page < startPage + cap; page++) {141 if (ctx.signal?.aborted) return;142 const { url, res, parsed } = await this.fetchSalePage(ctx, sale, page);143 if (!parsed) {144 ctx.anomaly(page === 1 ? 'sale_parse_failed' : 'pagination_failure', `${sale.id} p${page}: ${res.error ?? res.httpStatus}`);145 return;146 }147 if (parsed.sale) Object.assign(sale, { date: parsed.sale.date ?? sale.date, location: parsed.sale.location ?? sale.location, title: parsed.sale.title ?? sale.title, extra: { ...sale.extra, ...(parsed.sale.extra ?? {}) } });148 if (parsed.lots.length === 0) {149 if (page === 1) ctx.anomaly('selector_missing', `${sale.id}: no lots parsed`);150 return;151 }152 const payload: SaleResultsPayload = { kind: 'sale_results', url, sale: { ...sale }, page, totalLots: parsed.totalLots, lots: parsed.lots };153 yield { url, externalId: `${sale.id}:p${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };154 if (onPage) await onPage(page, parsed.lots.length);155 if (!parsed.hasMore) return;156 }157 ctx.anomaly('pagination_failure', `${sale.id}: page cap ${cap} reached`);158 }159160 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {161 const cursor = (ctx.options.cursor ?? {}) as Cursor;162 // keep sales that have started (timed sales expose their end date in extra.end_date → wait for it)163 const sales = (await this.listSales(ctx)).filter((s) => {164 const when = typeof s.extra.end_date === 'string' ? s.extra.end_date : s.date;165 return !when || new Date(when).getTime() <= Date.now() + 86_400_000;166 });167 if (sales.length === 0) {168 ctx.anomaly('past_list_failed', 'no past sales found on the index');169 return;170 }171 if (ctx.options.mode === 'backfill') {172 yield* this.backfill(ctx, sales, cursor);173 return;174 }175 sales.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? ''));176 const done = new Set<string>(cursor.done ?? []);177 // sales seen without any realised price yet (still running / results not published) → re-check after a few days178 const pending: Record<string, string> = { ...(cursor.pending ?? {}) };179 const recheckMs = Number(this.meta.config.pendingRecheckDays ?? 3) * 86_400_000;180 const maxUnfinished = Number(this.meta.config.maxUnfinishedPerRun ?? Math.max(this.salesPerRun * 3, 12));181 let processed = 0;182 let unfinished = 0;183 let count = 0;184 for (const sale of sales) {185 if (ctx.signal?.aborted || this.reached(ctx, count) || processed >= this.salesPerRun || unfinished >= maxUnfinished) break;186 if (done.has(sale.id)) continue;187 const seenAt = pending[sale.id] ? new Date(pending[sale.id]!).getTime() : 0;188 if (seenAt && Date.now() - seenAt < recheckMs && ctx.options.mode !== 'probe') continue;189 let pages = 0;190 let complete = true;191 for await (const raw of this.crawlSale(ctx, sale)) {192 pages++;193 count++;194 yield raw;195 // A first page without a single realised price = the sale is still running or results are not published yet:196 // keep its lots as auction_lot records but do not paginate further and do not mark the sale done.197 if (pages === 1 && !(raw.payload as SaleResultsPayload).lots.some((l) => l.sold)) {198 complete = false;199 unfinished++;200 break;201 }202 if (this.reached(ctx, count)) break;203 }204 if (!complete) {205 pending[sale.id] = new Date().toISOString();206 await ctx.setCursor({ done: [...done].slice(-500), pending: trimPending(pending) });207 continue;208 }209 processed++;210 if (pages > 0 && !this.reached(ctx, count)) {211 done.add(sale.id);212 delete pending[sale.id];213 await ctx.setCursor({ done: [...done].slice(-500), pending: trimPending(pending) });214 }215 }216 }217218 /** Backfill: every past sale, oldest first, resumable at (sale index, page); ends with {finished:true}. */219 protected async *backfill(ctx: CrawlContext, sales: SaleRef[], cursor: Cursor): AsyncIterable<RawRecordInput> {220 if (cursor.finished) return;221 sales.sort((a, b) => (a.date ?? '').localeCompare(b.date ?? '') || a.id.localeCompare(b.id));222 let index = cursor.backfill?.index ?? 0;223 let itemsProcessed = cursor.backfill?.itemsProcessed ?? 0;224 let startPage = cursor.backfill?.page ?? 1;225 let fetched = 0;226 const maxPages = this.policy.backfillMaxPages;227 for (; index < sales.length; index++, startPage = 1) {228 const sale = sales[index]!;229 let stop = false;230 for await (const raw of this.crawlSale(ctx, sale, startPage, async (page, lots) => {231 itemsProcessed += lots;232 fetched++;233 await ctx.setCursor({ backfill: { index, page: page + 1, itemsProcessed } });234 await ctx.progress({ page: index + 1, totalPages: sales.length, itemsProcessed, reachedDate: sale.date ? new Date(sale.date) : null, cursor: { index, page: page + 1 } });235 if (fetched >= maxPages || ctx.signal?.aborted) stop = true;236 })) {237 yield raw;238 if (stop) return;239 }240 await ctx.setCursor({ backfill: { index: index + 1, page: 1, itemsProcessed } });241 }242 await ctx.setCursor({ finished: true, backfill: { index: sales.length, page: 1, itemsProcessed } });243 await ctx.progress({ page: sales.length, totalPages: sales.length, itemsProcessed, cursor: { finished: true } });244 }245246 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {247 const p = SaleResultsPayloadSchema.parse(raw.payload);248 const out: NormalizedRecord[] = [];249 const saleDate = p.sale.date ? new Date(p.sale.date) : null;250 for (const lot of p.lots) {251 if (this.house.requireCurrency && !lot.currency) continue;252 const slug = this.categoryFor(p.sale, lot);253 if (!slug) continue;254 const text = `${lot.title} ${lot.subtitle ?? ''}`.trim();255 const g = parseGradeFromTitle(text);256 const isWatch = ['rolex', 'omega', 'patek_philippe', 'audemars_piguet', 'other_watches'].includes(slug);257 const attributes = AssetAttributesSchema.parse({258 categorySlug: slug,259 name: lot.title,260 model: lot.subtitle,261 brand: brandFromSlug(slug, text),262 // "réf. 5513" / "Ref 16233" — normalise the French/German abbreviation before the English reference parser263 reference: isWatch ? watchReference(text) ?? watchReference(text.replace(/\br[ée]f(?:[ée]rence|erenz)?\.?\s*/gi, 'Ref. ')) : null,264 year: yearFromTitle(text),265 identifiers: { [this.house.idKey]: `${p.sale.id}/${lot.lotNo}` },266 metadata: { sale_id: p.sale.id, sale_title: p.sale.title, sale_url: p.sale.url, estimate_low: lot.estimateLow, estimate_high: lot.estimateHigh, ...p.sale.extra, ...lot.extra },267 });268 const base = {269 connectorId: this.meta.id,270 sourceId: this.meta.sourceId,271 sourceUrl: lot.url,272 externalId: `${p.sale.id}:${lot.lotNo}`,273 rawTitle: lot.subtitle ? `${lot.title} — ${lot.subtitle}` : lot.title,274 description: lot.description,275 imageUrls: lot.image ? [lot.image] : [],276 attributes,277 grade: { grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, qualifier: null, certificationNumber: null },278 condition: { condition: null, conditionRaw: null, completeness: null },279 observedAt: raw.fetchedAt,280 confidence: this.confidenceFor(slug, lot),281 parserVersion: this.parserVersion,282 };283 const currency = (lot.currency ?? this.house.defaultCurrency) as CurrencyCode;284 const lotDate = lot.date ? new Date(lot.date) : saleDate;285 const location = p.sale.location ?? this.house.location;286 if (lot.sold && lot.price && lotDate && !Number.isNaN(lotDate.getTime())) {287 out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate: lotDate, price: lot.price, currency, buyerPremiumIncluded: lot.premiumIncluded ?? this.house.premiumIncluded, quantity: 1, isBundle: isBundleMultilingual(text), location, auctionHouse: this.house.houseName, lotNumber: lot.lotNo }));288 } else {289 const status = lotDate && lotDate.getTime() < raw.fetchedAt.getTime() ? 'ended' : lotDate ? 'upcoming' : 'unknown';290 out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: this.house.houseName, auctionName: p.sale.title, lotNumber: lot.lotNo, startsAt: lotDate, endsAt: lotDate, estimateLow: lot.estimateLow, estimateHigh: lot.estimateHigh, currentBid: lot.sold ? lot.price : null, currency, status, location }));291 }292 }293 return out;294 }295296 protected confidenceFor(_slug: string, _lot: ParsedLot): number {297 return 0.85;298 }299}300301/** Shared category resolution (see `SaleResultsConnector.categoryFor`). Exported for connectors with extra rules. */302export function resolveCategory(saleLabel: string, lot: ParsedLot, fallbackSlug: string | null): string | null {303 const text = `${lot.title} ${lot.subtitle ?? ''}`.trim();304 const full = `${text} ${lot.description ?? ''}`.slice(0, 500);305 const strong = strongLotHint(full);306 const saleHint: DeptHint = hintFromLabel(englishLabel(saleLabel));307 const hint: DeptHint = strong ?? saleHint;308 if (hint === 'wine') {309 if (/whisky|whiskey|威士忌|ウイスキー|bourbon|macallan|yamazaki|山崎|hibiki|響|yoichi|余市|karuizawa|輕井澤|軽井沢|springbank|bowmore|ardbeg|glenfiddich|dalmore|laphroaig|brora|port ellen/i.test(full)) return 'whisky';310 if (/cognac|armagnac|calvados|干邑|白蘭地|ブランデー/i.test(full)) return 'cognac';311 if (/\brum\b|\brhum\b|朗姆|ラム酒/i.test(full)) return 'rum';312 return 'wine';313 }314 if (hint === 'asian' || hint === 'antiquities') return slugFromTitle(text, hint) ?? 'antiques';315 if (hint === 'cars') {316 if (/\b(helmet|casque|helm|poster|affiche|plakat|trophy|trophée|suit|combinaison|gloves|gants|photograph|photographie|model|maquette|modell|miniature|book|livre|buch|sign|plaque|enamel|programme|program|watch|montre|mascot|mascotte|badge)\b/i.test(full) && !/\b(chassis|châssis|fahrgestell|telaio|\bvin\b|immatricul|registration|kilom|mileage)\b/i.test(full)) return 'automotive_memorabilia';317 return slugFromTitle(full, 'cars') ?? 'automobiles';318 }319 return slugFromTitle(text, hint) ?? slugFromTitle(full, hint) ?? (hint !== 'unknown' ? slugFromTitle('', hint) : null) ?? (saleHint !== 'unknown' ? slugFromTitle('', saleHint) : null) ?? fallbackSlug;320}321322/** Unescape HTML entities commonly found in server-rendered auction pages. */323export function decodeEntities(s: string): string {324 return s325 .replace(/ | /g, ' ')326 .replace(/&/g, '&')327 .replace(/</g, '<')328 .replace(/>/g, '>')329 .replace(/"/g, '"')330 .replace(/'|'|’|’/g, "'")331 .replace(/€/g, '€')332 .replace(/£/g, '£')333 .replace(/é/g, 'é')334 .replace(/è/g, 'è')335 .replace(/à/g, 'à')336 .replace(/ç/g, 'ç')337 .replace(/ô/g, 'ô')338 .replace(/ê/g, 'ê')339 .replace(/ü/g, 'ü')340 .replace(/ö/g, 'ö')341 .replace(/ä/g, 'ä')342 .replace(/ß/g, 'ß')343 .replace(/º/g, 'º')344 .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n)))345 .replace(/&#x([0-9a-f]+);/gi, (_, h: string) => String.fromCodePoint(Number.parseInt(h, 16)));346}347348/** Text content of an HTML fragment (tags stripped, entities decoded, whitespace collapsed). */349export function textOf(fragment: string | null | undefined): string {350 if (!fragment) return '';351 return decodeEntities(fragment.replace(/<br\s*\/?>/gi, ' ').replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();352}353354/** Split an HTML document into item chunks starting at each match of `startRe` (the chunk runs to the next match). */355export function chunksBetween(htmlText: string, startRe: RegExp, endBoundary?: RegExp): string[] {356 const re = new RegExp(startRe.source, startRe.flags.includes('g') ? startRe.flags : `${startRe.flags}g`);357 const idx: number[] = [];358 let m: RegExpExecArray | null;359 while ((m = re.exec(htmlText))) idx.push(m.index);360 if (idx.length === 0) return [];361 let end = htmlText.length;362 if (endBoundary) {363 const tail = htmlText.slice(idx[idx.length - 1]!);364 const e = tail.search(endBoundary);365 if (e > 0) end = idx[idx.length - 1]! + e;366 }367 return idx.map((s, k) => htmlText.slice(s, idx[k + 1] ?? end)).filter((c) => c.length > 0);368}369370/** First capture group of `re` in `s`, entity-decoded and trimmed; null when absent. */371export function pick(s: string, re: RegExp): string | null {372 const m = s.match(re);373 return m ? textOf(m[1] ?? m[0]) || null : null;374}375376export function absolute(base: string, href: string | null | undefined): string | null {377 if (!href) return null;378 try {379 return new URL(decodeEntities(href), base).toString();380 } catch {381 return null;382 }383}384