TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { clean, isNumisBundle, numisAttributes, numisCategory, parseAuctionDate, parseCoinGrade, realized, type NumisSlug } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * Spink & Son (London / New York / Hong Kong / Singapore) — coins, banknotes, stamps, medals, bonds.9 * Static Yii/PHP pages: /auctions/prices carries a <select> of every sale since December 2002 (code,10 * date, title); /auction/<code>?page=1 lists the sale's lots (links + short description, no price);11 * /lot/<code><lot 6 digits> shows the full description, "Sold for £X" and "Starting price". Spink does12 * not state on the lot page whether "Sold for" includes the buyer's premium → buyer_premium_included=null.13 */1415const SITE = 'https://www.spink.com';16const PARSER_VERSION = '1.0.0';1718export const SaleSchema = z.object({ code: z.string(), title: z.string(), dateText: z.string().nullable() });19export const PayloadSchema = z.object({20 kind: z.literal('lot_page'),21 sale: SaleSchema,22 url: z.string(),23 lotNumber: z.string(),24 headline: z.string().nullable(),25 description: z.string(),26 soldText: z.string().nullable(),27 startingText: z.string().nullable(),28 images: z.array(z.string()),29});30export type Payload = z.infer<typeof PayloadSchema>;3132/** /auctions/prices → sales from the auction dropdown ("15 May 2003 - No 3024 - The Slaney Collection…"). */33export function parseSalesDropdown(htmlText: string): z.infer<typeof SaleSchema>[] {34 const $ = H.load(htmlText);35 const out: z.infer<typeof SaleSchema>[] = [];36 $('select[name="auction"] option').each((_, o) => {37 const code = ($(o).attr('value') ?? '').trim();38 const label = clean($(o).text());39 if (!code || code === '-1' || !label) return;40 const m = label.match(/^(\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4})\s*-\s*No\s+([A-Za-z0-9]+)\s*-\s*(.+)$/);41 if (!m) return;42 if (out.some((s) => s.code === code)) return;43 out.push({ code, title: m[3]!.trim(), dateText: m[1]! });44 });45 return out;46}4748/** /auction/<code>?page=N → lot URLs (the category accordions list every lot of the sale). */49export function parseLotLinks(htmlText: string, code: string): string[] {50 const re = new RegExp(`href="(/lot/${code}\\d{6})"`, 'g');51 const out = new Set<string>();52 let m: RegExpExecArray | null;53 while ((m = re.exec(htmlText))) out.add(`${SITE}${m[1]}`);54 return [...out];55}5657/** Lot page → payload (null when the page is not a lot page). */58export function parseLotPage(htmlText: string, url: string, sale: z.infer<typeof SaleSchema>): Payload | null {59 const $ = H.load(htmlText);60 const id = clean($('span.id').first().text()).match(/Lot:\s*([A-Za-z0-9]+)/)?.[1] ?? url.match(/\/lot\/[A-Za-z0-9]+?(\d{6})$/)?.[1]?.replace(/^0+/, '');61 const descEl = $('span.description').first();62 if (!id || !descEl.length) return null;63 const headline = clean(descEl.find('b').first().text()) || null;64 const description = clean(descEl.html() ?? '').replace(/Subject to \d+% VAT on Buyer.?s Premium\..*$/i, '').replace(/https?:\/\/\S+/g, ' ').replace(/\s+/g, ' ').trim();65 let soldText: string | null = null;66 let startingText: string | null = null;67 $('span.sold-for').each((_, el) => {68 const label = clean($(el).text()).toLowerCase();69 const amount = clean($(el).parent().find('span.amount').first().text()) || null;70 if (label.startsWith('sold for')) soldText = amount;71 else if (label.startsWith('starting price')) startingText = amount;72 });73 const images = $('img.image.img-responsive, .thumbnail-inner img').map((_, img) => $(img).attr('src') ?? '').get().filter((s) => /cloudfront\.net\/auction\//.test(s));74 const name = clean($('span.name').first().text());75 const saleFromPage = name.match(/Auction:\s*([A-Za-z0-9]+)\s*-\s*(.+)$/);76 return {77 kind: 'lot_page',78 sale: { code: sale.code, title: saleFromPage?.[2]?.trim() || sale.title, dateText: sale.dateText },79 url,80 lotNumber: id,81 headline,82 description,83 soldText,84 startingText,85 images: [...new Set(images)].slice(0, 4),86 };87}8889/** Sale title → department slug; null = not numismatic/philatelic (skipped). */90export function spinkDepartment(title: string): NumisSlug | null {91 const t = title.toLowerCase();92 if (/wine|spirit|whisky|boozing|jewel|sapphire|handbag|watch|autograph|book|manuscript|map|bond|share|scripophily|ephemera only|test cross-listed/.test(t) && !/coin|banknote|stamp|medal/.test(t)) return null;93 if (/banknote|paper money|currency/.test(t) && !/coin|stamp/.test(t)) return 'banknotes';94 if (/stamp|philatel|postal/.test(t) && !/coin|banknote/.test(t)) return 'stamps';95 if (/orders, decorations|medals and militaria|medal/.test(t) && !/coin|banknote|stamp/.test(t)) return 'medals';96 if (/coin|numismat|sovereign|gold|silver|ancient|celtic|british|world/.test(t)) return 'coins';97 return null;98}99100interface Cursor {101 doneSales?: string[];102 inProgress?: { sale: z.infer<typeof SaleSchema>; lotUrls: string[]; index: number } | null;103 done?: boolean;104 updatedAt?: string;105}106107export class SpinkConnector extends BaseConnector {108 readonly version = '1.0.0';109 readonly parserVersion = PARSER_VERSION;110 protected override minIntervalMs = 1500;111 override readonly urlPatterns = [/spink\.com\/lot\/[A-Za-z0-9]+/i];112113 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {114 const lotsPerRun = Number(this.meta.config.lotsPerRun ?? 80);115 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };116 const done = new Set(cursor.doneSales ?? []);117 const text = (url: string, extra: Parameters<CrawlContext['fetch']>[1] = {}) => ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0, ...extra });118 let fetched = 0;119 let yielded = 0;120121 let state = cursor.inProgress ?? null;122 if (!state) {123 await this.throttle();124 const prices = await text(`${SITE}/auctions/prices`);125 fetched++;126 const sales = prices.success && prices.html ? parseSalesDropdown(prices.html) : [];127 if (!sales.length) {128 ctx.anomaly(prices.success ? 'selector_missing' : 'page_fetch_failed', `/auctions/prices: ${prices.error ?? prices.httpStatus ?? 'no auction dropdown'}`);129 return;130 }131 const now = Date.now();132 const closed = sales.filter((s) => spinkDepartment(s.title) !== null && (parseAuctionDate(s.dateText)?.getTime() ?? Infinity) < now - 3 * 86_400_000);133 // Recent sales redirect (302) until Spink publishes the results; try a few candidates per run.134 let attempts = 0;135 for (const next of closed) {136 if (done.has(next.code) || attempts >= 5 || ctx.signal?.aborted) continue;137 attempts++;138 await this.throttle();139 const list = await text(`${SITE}/auction/${next.code}?page=1`);140 fetched++;141 const lotUrls = list.success && list.html ? parseLotLinks(list.html, next.code) : [];142 if (!lotUrls.length) {143 const redirected = list.finalUrl && !list.finalUrl.includes(`/auction/${next.code}`);144 ctx.anomaly(redirected ? 'pagination_failure' : list.success ? 'selector_missing' : 'page_fetch_failed', `/auction/${next.code}: ${redirected ? `redirected to ${list.finalUrl}` : (list.error ?? list.httpStatus ?? 'no lot links')}`);145 if (list.success && !redirected) done.add(next.code); // genuinely empty sale page; redirected sales are retried later146 continue;147 }148 state = { sale: next, lotUrls, index: 0 };149 if (ctx.options.mode === 'backfill') await ctx.progress({ page: closed.findIndex((s) => s.code === next.code) + 1, totalPages: closed.length, itemsProcessed: 0, reachedDate: parseAuctionDate(next.dateText) });150 break;151 }152 if (!state) {153 if (ctx.options.mode === 'backfill' && closed.every((s) => done.has(s.code))) await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-2000), done: true, updatedAt: new Date().toISOString() });154 else await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-2000), inProgress: null, updatedAt: new Date().toISOString() });155 return;156 }157 }158159 while (state.index < state.lotUrls.length && fetched < lotsPerRun + 2 && !ctx.signal?.aborted && !this.reached(ctx, yielded)) {160 const url = state.lotUrls[state.index]!;161 await this.throttle();162 const res = await text(url, {163 expect: ['title', 'price'],164 parse: (r) => {165 const p = r.html ? parseLotPage(r.html, url, state!.sale) : null;166 return p ? { title: p.description, price: p.soldText ?? p.startingText } : null;167 },168 });169 fetched++;170 state.index++;171 const payload = res.success && res.html ? parseLotPage(res.html, url, state.sale) : null;172 if (!payload) {173 ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);174 } else if (payload.soldText) {175 yielded++;176 yield { url, externalId: url.replace(`${SITE}/lot/`, ''), kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };177 }178 if (state.index % 10 === 0) await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-2000), inProgress: state, updatedAt: new Date().toISOString() });179 }180 if (state.index >= state.lotUrls.length) {181 done.add(state.sale.code);182 state = null;183 }184 await ctx.setCursor({ ...cursor, doneSales: [...done].slice(-2000), inProgress: state, updatedAt: new Date().toISOString() });185 }186187 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {188 const p = PayloadSchema.parse(raw.payload);189 const price = realized(p.soldText);190 const saleDate = parseAuctionDate(p.sale.dateText);191 if (!price || !saleDate) return [];192 const fallback = spinkDepartment(p.sale.title) ?? 'coins';193 const categorySlug = fallback === 'stamps' ? 'stamps' : numisCategory(p.description, p.sale.title, fallback);194 const g = parseCoinGrade(p.description);195 const cert = p.description.match(/Cert\.?\s*#\s*([\d-]{6,15})/i)?.[1] ?? g.certificationNumber;196 const start = realized(p.startingText, price.currency);197 const identifiers: Record<string, string> = { spink_lot: `${p.sale.code}-${p.lotNumber}` };198 if (cert && g.grader) identifiers[`${g.grader}_cert`] = cert;199 const attributes = numisAttributes({200 categorySlug,201 title: p.headline ? `${p.headline} — ${p.description}` : p.description,202 section: p.sale.title,203 identifiers,204 metadata: { sale_code: p.sale.code, sale_title: p.sale.title, headline: p.headline, starting_price: start?.amount ?? null, sold_for: price.amount, buyer_premium: "unknown — Spink prints 'Sold for' without stating whether the buyer's premium is included", spink_reference: p.description.match(/\b(S\.\s?[A-Z]{0,2}\d{2,4}[A-Z]?|Pick\s?\d+[a-z]?|P\d{1,4}[a-z]?|SG\s?\d+[a-z]?)\b/)?.[1] ?? null },205 });206 attributes.name = p.headline ? `${p.headline} — ${p.description.slice(0, 160)}` : p.description.slice(0, 200);207 const sale = makeSale({208 meta: this.meta,209 sourceUrl: p.url,210 externalId: `${p.sale.code}-${p.lotNumber}`,211 rawTitle: p.description.length > 240 ? `${p.description.slice(0, 239)}…` : p.description,212 description: p.description,213 attributes,214 price: price.amount,215 currency: price.currency,216 saleDate,217 buyerPremiumIncluded: null,218 auctionHouse: 'Spink',219 lotNumber: p.lotNumber,220 imageUrls: p.images,221 observedAt: raw.fetchedAt,222 parserVersion: PARSER_VERSION,223 confidence: g.grader ? 0.88 : 0.8,224 isBundle: isNumisBundle(p.description) || /\[\d+ notes?\]|\(\d{2,}\)/.test(p.description),225 conditionRaw: g.conditionRaw,226 location: price.currency === 'HKD' ? 'HK' : price.currency === 'SGD' ? 'SG' : price.currency === 'USD' ? 'US' : 'GB',227 });228 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: cert };229 return [sale];230 }231}232233export default (meta: ConnectorMeta) => new SpinkConnector(meta);234