TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateWords, lotAttributes, makeSale } from '../../firecrawl/_carlib/index.js';5import { safeYear } from '../_auction-lib/categories.js';67/**8 * Daniel F. Kelleher Auctions — prices realized hosted on Stamp Auction Network (SAN).9 * Plain HTTPS. See meta.json accessNotes.10 */1112const SAN = 'https://StampAuctionNetwork.com';13const PARSER_VERSION = '1.0.0';14const MON: Record<string, number> = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };1516export const SaleSchema = z.object({ sale: z.string(), title: z.string(), dateText: z.string().nullable(), url: z.string() });17export const LotSchema = z.object({18 lotNumber: z.string(),19 catNo: z.string().nullable(),20 headline: z.string().nullable(),21 description: z.string(),22 image: z.string().nullable(),23 soldText: z.string().nullable(),24 closedText: z.string().nullable(),25 soldFor: z.string().nullable(),26});27export const PayloadSchema = z.object({ kind: z.literal('category_page'), sale: SaleSchema, majorGroup: z.string(), category: z.string(), url: z.string(), lots: z.array(LotSchema) });28export type Payload = z.infer<typeof PayloadSchema>;2930function clean(s: string): string {31 return s.replace(/&/g, '&').replace(/¢/g, '¢').replace(/�?39;|'/g, "'").replace(/"/g, '"').replace(/ /g, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();32}3334/**35 * Firm page (Kelleher.cfm) or the SAN archive (pr.cfm) → Kelleher sales in the cCatalog format.36 * When the page has a "PricesRealized" anchor only the section after it is used (closed sales).37 */38export function parseFirmPage(html: string): z.infer<typeof SaleSchema>[] {39 const out: z.infer<typeof SaleSchema>[] = [];40 const anchor = html.search(/NAME\s*=\s*"PricesRealized"/i);41 const scope = anchor >= 0 ? html.slice(anchor) : html;42 const re = /Kelleher Auctions[^<]{0,60}(?:<\/strong>)?\s*(?:<\/font>)?\s*<A HREF="(https?:\/\/StampAuctionNetwork\.com\/cCatalog\.cfm\?SrchFirm=V&(?:amp;)?SrchSale=(\d+))">\s*([\s\S]*?)<\/A>/gi;43 let m: RegExpExecArray | null;44 while ((m = re.exec(scope))) {45 if (out.some((s) => s.sale === m![2])) continue;46 const label = clean(m[3]!);47 const dm = label.match(/-\s*([A-Za-z]+ \d{1,2}(?:\s*[-–]\s*\d{1,2})?,\s*\d{4})\s*$/);48 out.push({ sale: m[2]!, title: dm ? label.slice(0, dm.index).trim() : label, dateText: dm ? dm[1]! : null, url: m[1]!.replace(/&/g, '&') });49 }50 return out;51}5253export function parseGroupLinks(html: string, level: 2 | 3): string[] {54 const re = level === 2 ? /href="(cCatalog2\.cfm\?[^"]+)"/gi : /href="(https?:\/\/StampAuctionNetwork\.com\/cCatalog3\.cfm\?[^"]+)"/gi;55 const out: string[] = [];56 let m: RegExpExecArray | null;57 while ((m = re.exec(html))) {58 const u = (m[1]!.startsWith('http') ? m[1]! : `${SAN}/${m[1]!}`).replace(/&/g, '&');59 if (!out.includes(u)) out.push(u);60 }61 return out;62}6364export function parseCategoryPage(html: string, sale: z.infer<typeof SaleSchema>, url: string): Payload {65 const q = new URL(url).searchParams;66 const majorGroup = q.get('MAJGROUP') ?? '';67 const category = q.get('CATDESCR') ?? '';68 const lots: z.infer<typeof LotSchema>[] = [];69 const tables = html.split(/<table id="CatTable"/i).slice(1);70 for (const t of tables) {71 const lotNumber = t.match(/Lot No:\s*(\d+)/)?.[1];72 if (!lotNumber) continue;73 const descCell = t.match(/bgcolor="EEEEEE"[^>]*>([\s\S]*?)<\/td>/i)?.[1] ?? '';74 const headline = descCell.match(/<B>([\s\S]*?)<\/B>/i)?.[1];75 const description = clean(descCell.replace(/<A HREF="[^"]*Photos[^"]*"[^>]*>[\s\S]*?<\/a>/gi, ' ').replace(/Suggested Bid.*$/i, ''));76 lots.push({77 lotNumber,78 catNo: t.match(/Cat No:\s*([^<\n]+)/)?.[1]?.trim() ?? null,79 headline: headline ? clean(headline) : null,80 description,81 image: t.match(/SRC="(https?:\/\/StampAuctionNetwork\.com\/Photos\/[^"]+)"/i)?.[1] ?? null,82 soldText: t.match(/Sold\.{2,}\s*([A-Z$]+\s?[\d,]+(?:\.\d{2})?)/)?.[1]?.trim() ?? null,83 closedText: t.match(/Closed\.{2,}\s*([A-Za-z]{3}-\d{1,2}-\d{4})/)?.[1] ?? null,84 soldFor: t.match(/Sold For\s+([\d,]+(?:\.\d{2})?)/)?.[1] ?? null,85 });86 }87 return { kind: 'category_page', sale, majorGroup, category, url, lots };88}8990export function closedDate(s: string | null): Date | null {91 const m = s?.match(/([A-Za-z]{3})-(\d{1,2})-(\d{4})/);92 if (!m) return null;93 const mo = MON[m[1]!.toLowerCase()];94 if (mo === undefined) return null;95 return new Date(Date.UTC(Number(m[3]), mo, Number(m[2])));96}9798export function kelleherCategory(majorGroup: string, text: string): string {99 const t = `${majorGroup} ${text}`.toLowerCase();100 if (/banknote|bank note|paper money|currency/.test(t)) return 'banknotes';101 if (/\bcoins?\b|numismat|medal\b/.test(t) && !/stamp|cover|postal/.test(t)) return 'coins';102 return 'stamps';103}104105export class KelleherConnector extends BaseConnector {106 readonly version = '1.0.0';107 readonly parserVersion = PARSER_VERSION;108 protected override minIntervalMs = 2000;109110 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {111 const salesPerRun = Number(this.meta.config.salesPerRun ?? 1);112 const pagesPerRun = Number(this.meta.config.categoryPagesPerRun ?? 30);113 const cursor = (ctx.options.cursor ?? {}) as { doneSales?: string[]; progress?: Record<string, string[]> };114 const done = new Set<string>(cursor.doneSales ?? []);115 const progress: Record<string, string[]> = cursor.progress ?? {};116 const firmUrl = `${SAN}/Kelleher.cfm`;117 await this.throttle();118 const firm = await ctx.fetch(firmUrl, { responseType: 'text', expect: ['title', 'date'], parse: (r) => ({ title: r.html ? parseFirmPage(r.html)[0]?.title ?? null : null, date: r.html ? parseFirmPage(r.html)[0]?.dateText ?? null : null }) });119 if (!firm.success || !firm.html) {120 ctx.anomaly('page_fetch_failed', `${firmUrl}: ${firm.error ?? firm.httpStatus}`);121 return;122 }123 let discovered = parseFirmPage(firm.html);124 if (ctx.options.mode === 'backfill') {125 await this.throttle();126 const archive = await ctx.fetch(`${SAN}/pr.cfm`, { responseType: 'text', minQuality: 0 });127 if (archive.success && archive.html) for (const s of parseFirmPage(archive.html)) if (!discovered.some((d) => d.sale === s.sale)) discovered.push(s);128 }129 // Only sales whose published date is in the past can have prices realized.130 discovered = discovered.filter((s) => !s.dateText || (dateWords(s.dateText)?.getTime() ?? 0) <= Date.now());131 const sales = discovered.filter((s) => !done.has(s.sale));132 let count = 0;133 let processed = 0;134 for (const sale of sales) {135 if (processed >= salesPerRun || ctx.signal?.aborted) break;136 await this.throttle();137 const top = await ctx.fetch(sale.url, { responseType: 'text', expect: ['title'], parse: (r) => ({ title: r.html && parseGroupLinks(r.html, 2).length ? sale.title : null }) });138 if (!top.success || !top.html) {139 ctx.anomaly('page_fetch_failed', `${sale.url}: ${top.error ?? top.httpStatus}`);140 break;141 }142 const seen = new Set(progress[sale.sale] ?? []);143 const pages: string[] = [];144 for (const g of parseGroupLinks(top.html, 2)) {145 if (ctx.signal?.aborted) break;146 await this.throttle();147 const grp = await ctx.fetch(g, { responseType: 'text', minQuality: 0 });148 if (grp.success && grp.html) pages.push(...parseGroupLinks(grp.html, 3));149 }150 const todo = pages.filter((p) => !seen.has(p));151 let n = 0;152 for (const p of todo) {153 if (n >= pagesPerRun || ctx.signal?.aborted || this.reached(ctx, count)) break;154 await this.throttle();155 const res = await ctx.fetch(p, {156 responseType: 'text',157 expect: ['title', 'price', 'date'],158 parse: (r) => {159 const f = r.html ? parseCategoryPage(r.html, sale, p).lots.find((l) => l.soldFor) : null;160 return f ? { title: f.headline ?? f.description, price: Number(f.soldFor!.replace(/,/g, '')), date: f.closedText } : null;161 },162 });163 n++;164 seen.add(p);165 if (!res.success || !res.html) {166 ctx.anomaly('page_fetch_failed', `${p}: ${res.error ?? res.httpStatus}`);167 continue;168 }169 const payload = parseCategoryPage(res.html, sale, p);170 if (payload.lots.length === 0) continue;171 count++;172 yield { url: p, externalId: `sale:${sale.sale}:${payload.majorGroup}:${payload.category}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };173 }174 progress[sale.sale] = [...seen];175 if (todo.length - n <= 0) {176 done.add(sale.sale);177 delete progress[sale.sale];178 processed++;179 }180 await ctx.setCursor({ doneSales: [...done].slice(-60), progress, updatedAt: new Date().toISOString() });181 if (todo.length - n > 0) break;182 }183 }184185 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {186 const p = PayloadSchema.parse(raw.payload);187 const out: NormalizedSale[] = [];188 for (const lot of p.lots) {189 if (!lot.soldFor) continue;190 const amount = Number(lot.soldFor.replace(/,/g, ''));191 const saleDate = closedDate(lot.closedText);192 if (!Number.isFinite(amount) || amount <= 0 || !saleDate) continue;193 const currency = /US\$|USD/.test(lot.soldText ?? 'US$') ? 'USD' : null;194 if (!currency) continue;195 const title = lot.headline ? `${lot.headline} ${lot.description.replace(lot.headline, '').trim()}`.trim() : lot.description;196 const url = `${p.url}#lot-${lot.lotNumber}`;197 const attributes = lotAttributes({198 categorySlug: kelleherCategory(p.majorGroup, title),199 name: lot.headline ?? lot.description.slice(0, 160),200 set: p.category || null,201 country: p.majorGroup || null,202 year: safeYear(lot.headline ?? lot.description),203 identifiers: { san_lot: `V-${p.sale.sale}-${lot.lotNumber}` },204 metadata: { sale: p.sale.sale, sale_title: p.sale.title, major_group: p.majorGroup, category: p.category, cat_no: lot.catNo, price_is_hammer: true },205 });206 out.push(207 makeSale({208 meta: this.meta,209 sourceUrl: url,210 externalId: `${p.sale.sale}-${lot.lotNumber}`,211 rawTitle: title.slice(0, 500),212 attributes,213 price: amount,214 currency: 'USD',215 saleDate,216 buyerPremiumIncluded: false,217 auctionHouse: 'Daniel F. Kelleher Auctions',218 lotNumber: lot.lotNumber,219 imageUrls: lot.image ? [lot.image] : [],220 description: lot.description,221 observedAt: raw.fetchedAt,222 parserVersion: PARSER_VERSION,223 isBundle: /\b(collection|accumulation|balance|group|lot of|selection|remainder)\b/i.test(title),224 location: 'US',225 }),226 );227 }228 return out;229 }230}231232export default (meta: ConnectorMeta) => new KelleherConnector(meta);233