TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../_carlib/index.js';5import { clean, isNumisBundle, numisAttributes, numisCategory, parseAuctionDate, parseCoinGrade, realized } from '../_g5-numismatics-lib/index.js';67/**8 * NumisBids — aggregator of numismatic auction catalogues (≈ 370 houses) with prices realized.9 * Public pages: /results (closed sales with a "Prices realized" link → sale id), /sale/<sid> (sale10 * header + category list), /sale/<sid>/category/<cid>?pg=N (100 lots per page with estimate, image,11 * truncated description and "Price realized: 160 USD"). Prices are hammer prices ("Buyer's premium is12 * not included" on the /pr/<sid> page). Engine: Firecrawl (Cloudflare refuses plain HTTPS).13 */1415const SITE = 'https://www.numisbids.com';16const PARSER_VERSION = '1.0.0';1718export const SaleSchema = z.object({ sid: z.string(), firm: z.string(), title: z.string(), dateText: z.string().nullable() });19export const CategorySchema = z.object({ cid: z.string(), name: z.string(), count: z.number().int().nullable() });20export const LotSchema = z.object({21 lotNumber: z.string(),22 url: z.string(),23 title: z.string(),24 estimateText: z.string().nullable(),25 realizedText: z.string().nullable(),26 image: z.string().nullable(),27});28export const PayloadSchema = z.object({29 kind: z.literal('category_page'),30 sale: SaleSchema,31 category: CategorySchema,32 url: z.string(),33 page: z.number().int(),34 totalPages: z.number().int(),35 lots: z.array(LotSchema),36});37export type Payload = z.infer<typeof PayloadSchema>;3839export interface ResultsEntry {40 sid: string;41 eventId: string | null;42 firm: string;43 title: string;44 subtitle: string | null;45 dayText: string | null;46}4748/** /results → closed sales that publish prices realized on NumisBids (newest first as listed). */49export function parseResultsPage(htmlText: string): ResultsEntry[] {50 const $ = H.load(htmlText);51 const out: ResultsEntry[] = [];52 $('tr').each((_, tr) => {53 const $tr = $(tr);54 const pr = $tr.find('a[href*="/pr/"]').first().attr('href');55 const sid = pr?.match(/\/pr\/(\d+)/)?.[1];56 if (!sid || out.some((e) => e.sid === sid)) return;57 const firm = $tr.find('td.firmcell img').attr('alt') ?? clean($tr.find('a.descr').first().text()).replace(/^-\s*/, '');58 const titleA = $tr.find('a[href*="/event/"] b').first();59 const title = clean(titleA.text());60 const eventId = $tr.find('a[href*="/event/"]').first().attr('href')?.match(/\/event\/(\d+)/)?.[1] ?? null;61 const subtitle = clean($tr.find('a.descr[href*="/event/"]').first().text()) || null;62 const dayText = clean($tr.find('.datetext').first().text()) || null;63 if (!title) return;64 out.push({ sid, eventId, firm: clean(firm), title, subtitle, dayText });65 });66 return out;67}6869export interface SalePage {70 sale: z.infer<typeof SaleSchema>;71 hasPrices: boolean;72 categories: z.infer<typeof CategorySchema>[];73}7475/** Sale header (firm, auction title, closing date) + category list, from any /sale/<sid>… page. */76export function parseSaleHeader(htmlText: string, sid: string): SalePage | null {77 const $ = H.load(htmlText);78 const status = $('.salestatus .text').first();79 const firm = clean(status.find('.name').first().text());80 const title = clean(status.find('b').first().text());81 if (!firm || !title) return null;82 const statusHtml = status.html() ?? '';83 const dateText = clean(statusHtml.match(/<\/b>\s*(?: |\s)*([^<]+)<br/i)?.[1] ?? '') || null;84 const hasPrices = status.find(`a[href*="/pr/${sid}"]`).length > 0 || /View prices realized/i.test(status.text());85 const categories: z.infer<typeof CategorySchema>[] = [];86 $(`a[href*="/sale/${sid}/category/"]`).each((_, a) => {87 const href = $(a).attr('href') ?? '';88 const cid = href.match(/\/category\/(\d+)/)?.[1];89 if (!cid || href.includes('?') || categories.some((c) => c.cid === cid)) return;90 const label = clean($(a).text());91 const m = label.match(/^(.*?)\s*\((\d+)\)\s*$/);92 if (!m) return; // navigation links to the same category without a count (e.g. "Go back to browse lots")93 categories.push({ cid, name: m[1]!.trim(), count: Number(m[2]) });94 });95 return { sale: { sid, firm, title, dateText }, hasPrices, categories };96}9798/** Category page → lots (100 per page) + "Page X of Y". */99export function parseCategoryPage(htmlText: string, sale: z.infer<typeof SaleSchema>, category: z.infer<typeof CategorySchema>, url: string): Payload {100 const $ = H.load(htmlText);101 const pg = $('.salenav .small').first().text().match(/Page\s+(\d+)\s+of\s+(\d+)/i);102 const lots: z.infer<typeof LotSchema>[] = [];103 $('div.browse').each((_, el) => {104 const e = $(el);105 const lotA = e.find('.lot a').first();106 const url = lotA.attr('href');107 const lotNumber = clean(lotA.text()).replace(/^Lot\s+/i, '');108 if (!url || !lotNumber) return;109 const title = clean(e.find('.summary a').first().text());110 if (!title) return;111 lots.push({112 lotNumber,113 url: url.startsWith('http') ? url : SITE + url,114 title,115 estimateText: clean(e.find('.estimate .rateclick').first().text()) || clean(e.find('.estimate').first().text().replace(/^Estimate:\s*/i, '')) || null,116 realizedText: clean(e.find('.realized .rateclick').first().text()) || clean(e.find('.realized').first().text().replace(/^Price realized:\s*/i, '')) || null,117 image: e.find('.browseimg img').attr('src') ?? null,118 });119 });120 return { kind: 'category_page', sale, category, url, page: pg ? Number(pg[1]) : 1, totalPages: pg ? Number(pg[2]) : 1, lots };121}122123interface Cursor {124 doneSales?: string[];125 inProgress?: { sid: string; sale: z.infer<typeof SaleSchema>; categories: z.infer<typeof CategorySchema>[]; catIndex: number; page: number } | null;126 /** backfill: next (older) sale id to inspect, descending */127 backfillSid?: number | null;128 minSeenSid?: number | null;129 done?: boolean;130 updatedAt?: string;131}132133export class NumisBidsConnector extends BaseConnector {134 readonly version = '1.0.0';135 readonly parserVersion = PARSER_VERSION;136 protected override minIntervalMs = 2500;137 override readonly urlPatterns = [/numisbids\.com\/sale\/\d+/i];138139 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {140 const salesPerRun = Number(this.meta.config.salesPerRun ?? 2);141 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 30);142 const skipSections = ((this.meta.config.skipSections as string[] | undefined) ?? []).map((s) => s.toLowerCase());143 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };144 const done = new Set(cursor.doneSales ?? []);145 let pages = 0;146 let yielded = 0;147 let salesFinished = 0;148149 // Quality hints per page type so the router does not escalate to Scrapfly on pages Firecrawl rendered fine.150 const fetchPage = (url: string) => {151 const isResults = /\/results(?:$|[?#])/.test(url);152 const isCategory = /\/category\//.test(url);153 return ctx.fetch(url, {154 engines: ['firecrawl', 'scrapfly'],155 expect: isCategory ? ['title', 'price'] : ['title'],156 parse: (r) => {157 if (!r.html) return null;158 if (isResults) {159 const entries = parseResultsPage(r.html);160 return entries.length ? { title: `${entries.length} sales with prices realized` } : null;161 }162 const sid = url.match(/\/sale\/(\d+)/)?.[1] ?? '0';163 const sale = parseSaleHeader(r.html, sid);164 if (!isCategory) return sale ? { title: sale.sale.title } : null;165 const lots = parseCategoryPage(r.html, sale?.sale ?? { sid, firm: '', title: '', dateText: null }, { cid: '0', name: '', count: null }, url).lots;166 const priced = lots.find((l) => l.realizedText) ?? lots[0];167 return priced ? { title: priced.title, price: priced.realizedText ?? priced.estimateText } : null;168 },169 });170 };171172 // Pick the sales to process: an unfinished one first, then new closed sales (incremental) or older ids (backfill).173 const queue: string[] = [];174 if (cursor.inProgress) queue.push(cursor.inProgress.sid);175 if (ctx.options.mode === 'backfill') {176 let sid = cursor.backfillSid ?? cursor.minSeenSid ?? null;177 if (sid === null) {178 await this.throttle();179 const res = await fetchPage(`${SITE}/results`);180 const entries = res.success && res.html ? parseResultsPage(res.html) : [];181 sid = entries.length ? Math.min(...entries.map((e) => Number(e.sid))) - 1 : null;182 }183 if (sid === null || sid <= 0) {184 await ctx.setCursor({ ...cursor, done: true, updatedAt: new Date().toISOString() });185 return;186 }187 for (let s = sid; s > 0 && queue.length < salesPerRun * 4; s--) if (!done.has(String(s))) queue.push(String(s));188 } else {189 await this.throttle();190 const res = await fetchPage(`${SITE}/results`);191 if (!res.success || !res.html) {192 ctx.anomaly('page_fetch_failed', `results: ${res.error ?? res.httpStatus}`);193 } else {194 const entries = parseResultsPage(res.html);195 if (!entries.length) ctx.anomaly('selector_missing', 'results page: no "Prices realized" rows');196 const sids = entries.map((e) => Number(e.sid)).filter((n) => Number.isFinite(n));197 if (sids.length) cursor.minSeenSid = Math.min(cursor.minSeenSid ?? Infinity, ...sids);198 for (const e of entries) if (!done.has(e.sid) && !queue.includes(e.sid)) queue.push(e.sid);199 }200 }201202 for (const sid of queue) {203 if (ctx.signal?.aborted || salesFinished >= salesPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break;204 let state = cursor.inProgress?.sid === sid ? cursor.inProgress : null;205 if (!state) {206 await this.throttle();207 const res = await fetchPage(`${SITE}/sale/${sid}`);208 pages++;209 const header = res.success && res.html ? parseSaleHeader(res.html, sid) : null;210 if (!header) {211 ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `sale ${sid}: ${res.error ?? res.httpStatus ?? 'no header'}`);212 if (ctx.options.mode === 'backfill' && res.success) await this.skipBackfillSale(ctx, cursor, done, sid); // removed / unpublished sale id213 continue;214 }215 if (!header.hasPrices || !header.categories.length) {216 // still open, or prices not published on NumisBids (house hosts them elsewhere)217 if (ctx.options.mode === 'backfill') await this.skipBackfillSale(ctx, cursor, done, sid);218 continue;219 }220 state = { sid, sale: header.sale, categories: header.categories.filter((c) => !skipSections.includes(c.name.toLowerCase())), catIndex: 0, page: 1 };221 }222 while (state.catIndex < state.categories.length) {223 if (ctx.signal?.aborted || pages >= pagesPerRun || this.reached(ctx, yielded)) break;224 const cat = state.categories[state.catIndex]!;225 const url = `${SITE}/sale/${sid}/category/${cat.cid}${state.page > 1 ? `?pg=${state.page}` : ''}`;226 await this.throttle();227 const res = await fetchPage(url);228 pages++;229 if (!res.success || !res.html) {230 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);231 break;232 }233 const payload = parseCategoryPage(res.html, state.sale, cat, url);234 if (!payload.lots.length) ctx.anomaly('parse_failure_page', `${url}: no lots parsed`);235 if (payload.lots.some((l) => l.realizedText)) {236 yielded++;237 yield { url, externalId: `sale:${sid}:cat:${cat.cid}:p${payload.page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };238 }239 if (payload.page < payload.totalPages) state.page++;240 else {241 state.catIndex++;242 state.page = 1;243 }244 cursor.inProgress = state;245 await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-300), updatedAt: new Date().toISOString() });246 }247 if (state.catIndex >= state.categories.length) {248 done.add(sid);249 salesFinished++;250 cursor.inProgress = null;251 if (ctx.options.mode === 'backfill') {252 cursor.backfillSid = Number(sid) - 1;253 await ctx.progress({ page: Number(sid), totalPages: null, itemsProcessed: yielded, reachedDate: parseAuctionDate(state.sale.dateText) });254 }255 await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-300), updatedAt: new Date().toISOString() });256 }257 }258 if (ctx.options.mode === 'backfill' && (cursor.backfillSid ?? 1) <= 0) {259 // The campaign walks sale ids downwards; id 1 is the oldest sale NumisBids hosts.260 await ctx.setCursor({ ...cursor, done: true, doneSales: [...done].slice(-300), updatedAt: new Date().toISOString() });261 }262 }263264 /** Backfill bookkeeping for a sale id that yields nothing (open sale, prices hosted elsewhere, removed id). */265 private async skipBackfillSale(ctx: CrawlContext, cursor: Cursor, done: Set<string>, sid: string): Promise<void> {266 done.add(sid);267 cursor.backfillSid = Number(sid) - 1;268 await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-300), updatedAt: new Date().toISOString() });269 }270271 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {272 const p = PayloadSchema.parse(raw.payload);273 const saleDate = parseAuctionDate(p.sale.dateText);274 if (!saleDate) return [];275 const out: NormalizedSale[] = [];276 const hint = `${p.category.name} ${p.sale.title}`;277 for (const lot of p.lots) {278 const price = realized(lot.realizedText);279 if (!price) continue; // unsold / withdrawn280 const categorySlug = numisCategory(lot.title, hint, 'coins');281 const g = parseCoinGrade(lot.title);282 const estimate = realized(lot.estimateText);283 const attributes = numisAttributes({284 categorySlug,285 title: lot.title,286 section: p.category.name,287 identifiers: { numisbids_lot: `${p.sale.sid}/${lot.lotNumber}` },288 metadata: { firm: p.sale.firm, auction: p.sale.title, numisbids_sale_id: p.sale.sid, numisbids_category_id: p.category.cid, estimate: estimate?.amount ?? null, estimate_currency: estimate?.currency ?? null, hammer_price: price.amount, buyer_premium: 'excluded (NumisBids prices realized are hammer prices)', title_truncated: /\.\.\.$/.test(lot.title) },289 });290 const sale = makeSale({291 meta: this.meta,292 sourceUrl: lot.url,293 externalId: `${p.sale.sid}-${lot.lotNumber}`,294 rawTitle: lot.title,295 attributes,296 price: price.amount,297 currency: price.currency,298 saleDate,299 buyerPremiumIncluded: false,300 auctionHouse: p.sale.firm,301 lotNumber: lot.lotNumber,302 imageUrls: lot.image ? [lot.image] : [],303 observedAt: raw.fetchedAt,304 parserVersion: PARSER_VERSION,305 confidence: g.grader ? 0.85 : 0.75,306 isBundle: isNumisBundle(lot.title, p.category.name),307 conditionRaw: g.conditionRaw,308 });309 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: g.certificationNumber };310 out.push(sale);311 }312 return out;313 }314}315316export default (meta: ConnectorMeta) => new NumisBidsConnector(meta);317