TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { hintFromLabel, slugFromTitle } from '../_auction-lib/categories.js';5import { amount, decodeEntities, houseCategory, isBundleTitle, lotAttributes, makeSale, parseUsDate, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js';67const SITE = 'https://www.doyle.com';8const HOUSE = 'Doyle';9const PARSER_VERSION = '1.0.0';10const PAGE_SIZE = 96;1112export const AuctionSchema = z.object({ auId: z.string(), code: z.string().nullable(), name: z.string(), dateText: z.string().nullable(), url: z.string() });13export const LotSchema = z.object({14 lotId: z.string(),15 lotNumber: z.string().nullable(),16 title: z.string(),17 url: z.string(),18 image: z.string().nullable(),19 soldText: z.string().nullable(),20 soldPrice: z.number().nullable(),21 estimateText: z.string().nullable(),22 estimateLow: z.number().nullable(),23 estimateHigh: z.number().nullable(),24 categoryId: z.string().nullable(),25});26export const PayloadSchema = z.object({ kind: z.literal('doyle_results_page'), auction: AuctionSchema, page: z.number(), hasNext: z.boolean(), lots: z.array(LotSchema) });27export type Payload = z.infer<typeof PayloadSchema>;28export type DoyleAuction = z.infer<typeof AuctionSchema>;2930/** /past-auctions/ → calendar items (newest first, as displayed). */31export function parsePastAuctions(htmlText: string): DoyleAuction[] {32 const $ = H.load(htmlText);33 const out: DoyleAuction[] = [];34 const seen = new Set<string>();35 $('.auction-calendar-item').each((_, el) => {36 const item = $(el);37 const href = item.find('a[href*="/auction/"][href*="au="]').first().attr('href');38 const auId = href?.match(/[?&]au=(\d+)/)?.[1];39 if (!href || !auId || seen.has(auId)) return;40 seen.add(auId);41 const name = H.text(item.find('h3, H3').first()) ?? '';42 // Live sales print "Date: …"; online-only sales print "Ends: …" (the close date = sale date).43 const dateText = H.text(item.find('strong').filter((_, s) => /^\s*(Date|Ends?):/i.test($(s).text())).first())?.replace(/^(Date|Ends?):\s*/i, '') ?? null;44 const code = href.match(/\/auction\/([0-9a-z]+)-/i)?.[1]?.toUpperCase() ?? null;45 out.push({ auId, code, name: decodeEntities(name), dateText, url: `${SITE}${href.split('#')[0]}` });46 });47 return out;48}4950/** "Mon d, yyyy hh:mm EST" header on an auction page (fallback when the calendar date is missing, e.g. seeds). */51export function parseAuctionHeaderDate(htmlText: string): string | null {52 const text = htmlText.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/g, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ');53 return text.match(/\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2}, 20\d{2}(?: \d{1,2}:\d{2} [A-Z]{2,4})?)\b/)?.[1] ?? null;54}5556function lotUrl(href: string): string {57 const u = new URL(href.replace(/&/g, '&'), SITE);58 const lot = u.searchParams.get('lot');59 const au = u.searchParams.get('au');60 const out = new URL(u.pathname, SITE);61 if (lot) out.searchParams.set('lot', lot);62 if (au) out.searchParams.set('au', au);63 return out.toString();64}6566/** One results page (/auction/search/?au=…&pp=96&pn=N&g=1) → lot cards. */67export function parseResultsPage(htmlText: string, auction: DoyleAuction, page: number): Payload {68 const $ = H.load(htmlText);69 const lots: Payload['lots'] = [];70 const seen = new Set<string>();71 $('div.auction-lot').each((_, el) => {72 const card = $(el);73 const link = card.find('p.auction-lot-title a').first();74 const href = link.attr('href') ?? card.find('a[href*="/auction/lot/"]').first().attr('href');75 const lotId = href?.match(/[?&]lot=(\d+)/)?.[1];76 if (!href || !lotId || seen.has(lotId)) return;77 seen.add(lotId);78 const span = card.find('span.lot-title').first();79 const parts = (span.html() ?? '').split(/<br\s*\/?>/i);80 const lotNumber = H.text(H.load(parts[0] ?? '')('body'))?.replace(/^Lot\s*/i, '').trim() || null;81 const title = decodeEntities(H.text(H.load(parts.slice(1).join(' '))('body')) ?? '') || decodeEntities(H.text(span) ?? '');82 if (!title) return;83 const categoryId = (span.attr('class') ?? '').match(/\bcat-(\d+)/)?.[1] ?? null;84 let soldText: string | null = null;85 let estimateText: string | null = null;86 card.find('strong').each((_, s) => {87 const t = H.text($(s)) ?? '';88 if (/^Sold for/i.test(t)) soldText = t;89 else if (/^Estimate/i.test(t)) estimateText = t;90 });91 const est = (estimateText ?? '').match(/\$([\d,]+)\s*-\s*\$([\d,]+)/);92 const img = card.find('img[src*="/stock/"]').first().attr('src') ?? null;93 lots.push({94 lotId,95 lotNumber,96 title,97 url: lotUrl(href),98 image: img,99 soldText,100 soldPrice: soldText ? amount((soldText as string).replace(/^Sold for/i, '')) : null,101 estimateText,102 estimateLow: est ? amount(est[1]) : null,103 estimateHigh: est ? amount(est[2]) : null,104 categoryId,105 });106 });107 const hasNext = new RegExp(`[?&](?:amp;)?pn=${page + 1}(?:&|'|"|$)`).test(htmlText);108 return { kind: 'doyle_results_page', auction, page, hasNext, lots };109}110111/** Doyle sale name + lot title → taxonomy slug (null when nothing confident). */112export function doyleCategory(saleName: string, title: string): string | null {113 const s = saleName.toLowerCase();114 // Natural history lots appear inside book/decorative sales; decide them before department fallbacks115 // (and before the handbag keyword "clutch" can misfire on "clutch of eggs").116 if (/\b(meteorite|pallasite|chondrite)\b/i.test(title)) return 'meteorites';117 if (/\b(fossil|ammonite|trilobite|dinosaur|sauropod|megalodon|mammoth|mosasaur|petrified|fossilized|coprolite)\b/i.test(title)) return 'fossils';118 if (/\b(mineral specimen|geode|quartz cluster|amethyst|tourmaline|fluorite|azurite|malachite|crystal cluster|agate slice)\b/i.test(title)) return 'minerals';119 if (/couture|handbag|fashion|luxury accessor/.test(s)) return slugFromTitle(title, 'fashion') ?? 'fashion_streetwear';120 if (/book|autograph|map|manuscript|bibliophil|librar|print(ed)? & manuscript/.test(s)) return slugFromTitle(title, 'books') ?? 'books';121 if (/photograph/.test(s)) return slugFromTitle(title, 'photographs') ?? 'photography';122 if (/jewel|gem/.test(s) && !/watch/.test(s)) return slugFromTitle(title, 'jewelry') ?? 'jewelry';123 if (/watch/.test(s)) return slugFromTitle(title, /\b(watch|wristwatch|chronograph|pocket watch)\b/i.test(title) ? 'watches' : 'jewelry') ?? 'jewelry';124 if (/coin|bank ?note|stamp|currency|numismat/.test(s)) return slugFromTitle(title, 'coins') ?? 'coins';125 // Silver-only sales; mixed sales ("… Furniture, Old Master Paintings, Silver") fall through to the title sweep below.126 if (/silver|vertu/.test(s) && !/furniture|painting|decorative|works of art/.test(s)) return slugFromTitle(title, 'silver') ?? 'silver';127 // Mixed sales ("English & Continental Furniture, Old Master Paintings, Silver"): let the title decide, default antiques.128 if (/furniture|decorative|works of art|at home|estate|collects/.test(s) && /painting|silver|art\b/.test(s)) {129 if (/\b(oil on|acrylic|watercolou?r|gouache|lithograph|etching|engraving|screenprint|woodcut|drawing|pastel|bronze|sculpture|mixed media)\b/i.test(title)) return slugFromTitle(title, 'art') ?? 'art';130 return slugFromTitle(title, 'furniture') ?? 'antiques';131 }132 if (/contemporary|post-war|modern art/.test(s)) return slugFromTitle(title, 'contemporary') ?? 'contemporary_art';133 if (/painting|prints|drawing|american art|european art|impressionist|old master|fine art|sculpture|works on paper|artist/.test(s)) return slugFromTitle(title, 'art') ?? 'art';134 if (/design|mid-century|20th century decorative/.test(s)) return slugFromTitle(title, 'design') ?? 'design_furniture';135 if (/asian|chinese|japanese|russian|furniture|decorative|english|continental|at home|estate|collects|interior|american story|americana|works of art|antique|rug|carpet/.test(s)) return slugFromTitle(title, 'furniture') ?? 'antiques';136 const generic = houseCategory(saleName, title, null);137 if (generic) return generic;138 return hintFromLabel(saleName) === 'unknown' ? null : 'antiques';139}140141/**142 * Doyle (New York) — auction results. Public server-rendered pages over plain HTTPS: /past-auctions/ (calendar of143 * closed sales with date) and /auction/search/?au=<id>&pp=96&pn=N&g=1 (result cards "Sold for $X", estimate).144 * robots.txt asks for crawl-delay 10, honoured. See meta.json accessNotes.145 */146export class DoyleConnector extends BaseConnector {147 readonly version = '1.0.0';148 readonly parserVersion = PARSER_VERSION;149 protected override minIntervalMs = 10_000; // robots.txt crawl-delay: 10150151 private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> {152 await this.throttle(url);153 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });154 if (!res.success || !res.html) {155 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);156 return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt };157 }158 return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt };159 }160161 private async listAuctions(ctx: CrawlContext): Promise<DoyleAuction[]> {162 if (ctx.options.seeds?.length) {163 const out: DoyleAuction[] = [];164 for (const s of ctx.options.seeds) {165 const auId = s.match(/[?&]au=(\d+)/)?.[1];166 if (!auId) continue;167 const code = s.match(/\/auction\/([0-9a-z]+)-/i)?.[1]?.toUpperCase() ?? null;168 out.push({ auId, code, name: '', dateText: null, url: s.startsWith('http') ? s : `${SITE}${s}` });169 }170 return out;171 }172 const r = await this.html(ctx, `${SITE}/past-auctions/`);173 if (!r.html) return [];174 const list = parsePastAuctions(r.html);175 if (!list.length) ctx.anomaly('selector_missing', 'past-auctions: no .auction-calendar-item found');176 return list;177 }178179 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {180 const auctionsPerRun = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.auctionsPerRun ?? 2);181 const maxPages = ctx.options.mode === 'probe' ? 1 : Number(this.meta.config.pagesPerAuction ?? 15);182 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);183 const all = await this.listAuctions(ctx);184 // Incremental: newest first (list order). Backfill: oldest first so progress walks the archive forward.185 const ordered = ctx.options.mode === 'backfill' ? [...all].reverse() : all;186 const pending = ordered.filter((a) => !done.has(a.auId));187 let processed = 0;188 let count = 0;189 let items = 0;190 for (const auction of pending) {191 if (ctx.signal?.aborted || processed >= auctionsPerRun || this.reached(ctx, count)) break;192 if (!auction.dateText) {193 // Seeds carry no calendar date: read the auction page header once.194 const page = await this.html(ctx, auction.url);195 if (page.html) {196 auction.dateText = parseAuctionHeaderDate(page.html);197 if (!auction.name) auction.name = decodeEntities(H.text(H.load(page.html)('h1').first()) ?? H.text(H.load(page.html)('title')) ?? '');198 }199 if (!auction.dateText) ctx.anomaly('missing_auction_date', auction.url);200 }201 let complete = true;202 for (let page = 1; page <= maxPages; page++) {203 if (ctx.signal?.aborted || this.reached(ctx, count)) {204 complete = false;205 break;206 }207 const url = `${SITE}/auction/search/?au=${auction.auId}&pp=${PAGE_SIZE}${page > 1 ? `&pn=${page}` : ''}&g=1`;208 const r = await this.html(ctx, url);209 if (!r.html) {210 complete = false;211 break;212 }213 const payload = parseResultsPage(r.html, auction, page);214 if (payload.lots.length === 0) {215 if (page === 1) ctx.anomaly('parse_failure_page', `${url}: no lot cards`);216 break;217 }218 count++;219 items += payload.lots.length;220 yield { url, externalId: `auction:${auction.auId}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt };221 if (!payload.hasNext) break;222 if (page === maxPages) complete = false;223 }224 processed++;225 if (complete) done.add(auction.auId);226 const doneList = [...done].slice(-500);227 await ctx.setCursor({ doneAuctions: doneList, updatedAt: new Date().toISOString() });228 await ctx.progress({ page: all.filter((a) => done.has(a.auId)).length, totalPages: all.length, itemsProcessed: items, cursor: { doneAuctions: doneList } });229 }230 if (ctx.options.mode === 'backfill' && all.length && all.every((a) => done.has(a.auId))) await ctx.setCursor({ doneAuctions: [...done].slice(-500), done: true, updatedAt: new Date().toISOString() });231 }232233 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {234 const p = PayloadSchema.parse(raw.payload);235 const saleDate = parseUsDate(p.auction.dateText);236 if (!saleDate || saleDate.getTime() > Date.now() + 86_400_000) return [];237 const out: NormalizedRecord[] = [];238 for (const l of p.lots) {239 if (!l.soldPrice) continue; // unsold / withdrawn / passed: no realised price240 const slug = doyleCategory(p.auction.name, l.title);241 if (!slug) continue;242 const g = saleGrade(l.title);243 const attributes = lotAttributes({244 categorySlug: slug,245 name: l.title,246 year: safeYear(l.title),247 identifiers: { doyle_lot_id: l.lotId },248 metadata: { auction_id: p.auction.auId, auction_code: p.auction.code, auction_name: p.auction.name, estimate_low: l.estimateLow, estimate_high: l.estimateHigh, doyle_category_id: l.categoryId },249 });250 out.push(251 makeSale({252 meta: this.meta,253 sourceUrl: l.url,254 externalId: l.lotId,255 rawTitle: l.title,256 attributes,257 price: l.soldPrice,258 currency: 'USD',259 saleDate,260 // Doyle lot pages print "Includes Buyer's Premium" under "Sold for".261 buyerPremiumIncluded: true,262 auctionHouse: HOUSE,263 lotNumber: l.lotNumber,264 imageUrls: l.image ? [l.image] : [],265 location: 'US',266 observedAt: raw.fetchedAt,267 parserVersion: PARSER_VERSION,268 grader: g.grader,269 grade: g.grade,270 isBundle: isBundleTitle(l.title),271 confidence: 0.85,272 }),273 );274 }275 return out;276 }277}278279export default (meta: ConnectorMeta) => new DoyleConnector(meta);280