TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../_carlib/index.js';5import { clean, isNumisBundle, numisAttributes, numisCategory, parseAuctionDate, parseCoinGrade, realized, type NumisSlug } from '../_g5-numismatics-lib/index.js';67/**8 * Noonans Mayfair (ex Dix Noonan Webb) — coins, banknotes, medals and tokens; archive of every sale9 * since 1991 with hammer prices. Public pages: /archive/past-catalogues/ (all catalogues, newest first)10 * → /archive/past-catalogues/<id>/catalogue/?layout=list&offset=N (40 lots per page: full description with11 * estimate, hammer price or "Unsold", lot id, image). Engine: Firecrawl (Cloudflare challenge on plain HTTPS).12 */1314const SITE = 'https://www.noonans.co.uk';15const PARSER_VERSION = '1.0.0';16const PAGE_SIZE = 40;1718export const AuctionSchema = z.object({ id: z.string(), title: z.string(), dateText: z.string().nullable() });19export const LotSchema = z.object({20 lotId: z.string(),21 lotNumber: z.string(),22 url: z.string(),23 description: z.string(),24 estimateText: z.string().nullable(),25 hammerText: z.string().nullable(),26 image: z.string().nullable(),27});28export const PayloadSchema = z.object({29 kind: z.literal('catalogue_page'),30 auction: AuctionSchema,31 url: z.string(),32 offset: z.number().int(),33 totalLots: z.number().int().nullable(),34 lots: z.array(LotSchema),35});36export type Payload = z.infer<typeof PayloadSchema>;3738/** Past-catalogues index → auctions (newest first). */39export function parsePastCatalogues(htmlText: string): z.infer<typeof AuctionSchema>[] {40 const $ = H.load(htmlText);41 const out: z.infer<typeof AuctionSchema>[] = [];42 $('.card-component--past-catalogues').each((_, el) => {43 const e = $(el);44 const href = e.find('a.card-component__full-card-link').attr('href') ?? '';45 const id = href.match(/past-catalogues\/(\d+)\//)?.[1];46 if (!id || out.some((a) => a.id === id)) return;47 const title = clean(e.find('.card-component__title').first().text());48 const dateText = clean(e.find('.card-component__text--secondary').first().text()) || null;49 if (title) out.push({ id, title, dateText });50 });51 return out;52}5354/** Catalogue list page (layout=list) → lots. */55export function parseCataloguePage(htmlText: string, auction: z.infer<typeof AuctionSchema>, url: string): Payload {56 const $ = H.load(htmlText);57 const offset = Number(url.match(/offset=(\d+)/)?.[1] ?? 0);58 const found = $('.pagination-top__filter-text--lower').first().text().match(/(\d[\d,]*)\s+lots?\s+found/i);59 const lots: z.infer<typeof LotSchema>[] = [];60 $('.card-component-list, .card-component--lot').each((_, el) => {61 const e = $(el);62 const a = e.find('a.card-component-list__full-card-link, a.card-component__full-card-link').first();63 const href = a.attr('href') ?? '';64 const lotId = href.match(/catalogue\/(\d+)\//)?.[1];65 if (!lotId || lots.some((l) => l.lotId === lotId)) return;66 const lotNumber = clean(e.find('.card-component__title--lot-number').first().text()).replace(/^№\s*/, '');67 const raw = clean(e.find('.card-component-list__info-text, .card-component-text--five-line').first().text());68 if (!lotNumber || !raw) return;69 // list layout ends the description with the estimate after a tab: "... Fine or better £90-£120"70 const em = raw.match(/\s((?:£|\$|€|HK\$|S\$|US\$)\s?[\d,.]+(?:\s*[-–]\s*(?:£|\$|€|HK\$|S\$|US\$)?\s?[\d,.]+)?)\s*$/);71 const estimateText = em ? em[1]!.trim() : null;72 const description = em ? raw.slice(0, em.index).trim() : raw;73 const hammerRaw = clean(e.find('.card-component__estimate').first().text());74 const hm = hammerRaw.match(/Hammer Price:\s*(.+)$/i);75 lots.push({76 lotId,77 lotNumber,78 url: `${SITE}/archive/past-catalogues/${auction.id}/catalogue/${lotId}/`,79 description,80 estimateText,81 hammerText: hm ? hm[1]!.trim() : null,82 image: e.find('img.card-component__image').attr('src') ?? null,83 });84 });85 return { kind: 'catalogue_page', auction, url, offset, totalLots: found ? Number(found[1]!.replace(/,/g, '')) : null, lots };86}8788/** Catalogue title → default taxonomy slug (null = not a numismatic sale, skipped). */89export function noonansDefaultSlug(title: string): NumisSlug | null {90 const t = title.toLowerCase();91 if (/banknote|paper money/.test(t)) return 'banknotes';92 if (/orders, decorations|medals and militaria|\bmilitaria\b|gallantry/.test(t)) return 'medals';93 if (/medallion|historical medal|token|ticket|passes/.test(t) && !/coin/.test(t)) return 'medals';94 if (/coin|numismatic|sovereign|celtic|roman|hammered|detectorist|treasure|collection of/.test(t)) return 'coins';95 if (/jewell|watch|silver|vertu|objects/.test(t)) return null;96 return null;97}9899interface Cursor {100 doneAuctions?: string[];101 inProgress?: { auction: z.infer<typeof AuctionSchema>; offset: number; totalLots: number | null } | null;102 done?: boolean;103 updatedAt?: string;104}105106export class NoonansConnector extends BaseConnector {107 readonly version = '1.0.0';108 readonly parserVersion = PARSER_VERSION;109 protected override minIntervalMs = 2500;110 override readonly urlPatterns = [/noonans\.co\.uk\/archive\/past-catalogues\/\d+/i];111112 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {113 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 1);114 const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 20);115 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };116 const done = new Set(cursor.doneAuctions ?? []);117 let pages = 0;118 let yielded = 0;119 let finished = 0;120121 const fetchPage = (url: string) => {122 const isIndex = /\/archive\/past-catalogues\/?(?:$|[?#])/.test(url);123 return ctx.fetch(url, {124 engines: ['firecrawl', 'scrapfly'],125 expect: isIndex ? ['title'] : ['title', 'price'],126 timeoutMs: isIndex ? 120_000 : undefined,127 parse: (r) => {128 if (!r.html) return null;129 if (isIndex) return { title: parsePastCatalogues(r.html)[0]?.title ?? null };130 const lot = parseCataloguePage(r.html, { id: '0', title: '', dateText: null }, url).lots[0];131 return lot ? { title: lot.description, price: lot.hammerText ?? lot.estimateText } : null;132 },133 });134 };135136 await this.throttle();137 const index = await fetchPage(`${SITE}/archive/past-catalogues/`);138 pages++;139 if (!index.success || !index.html) {140 ctx.anomaly('page_fetch_failed', `past-catalogues: ${index.error ?? index.httpStatus}`);141 return;142 }143 const all = parsePastCatalogues(index.html);144 if (!all.length) {145 ctx.anomaly('selector_missing', 'past-catalogues: no catalogue cards');146 return;147 }148 const numismatic = all.filter((a) => noonansDefaultSlug(a.title) !== null);149 const queue = [...(cursor.inProgress ? [cursor.inProgress.auction] : []), ...numismatic.filter((a) => !done.has(a.id) && a.id !== cursor.inProgress?.auction.id)];150 if (ctx.options.mode !== 'backfill') queue.splice(auctionsPerRun + 1); // incremental: newest sales only151152 for (const auction of queue) {153 if (ctx.signal?.aborted || finished >= auctionsPerRun || pages >= pagesPerRun || this.reached(ctx, yielded)) break;154 let state = cursor.inProgress?.auction.id === auction.id ? cursor.inProgress : { auction, offset: 0, totalLots: null as number | null };155 for (;;) {156 if (ctx.signal?.aborted || pages >= pagesPerRun || this.reached(ctx, yielded)) break;157 const url = `${SITE}/archive/past-catalogues/${auction.id}/catalogue/?layout=list&offset=${state.offset}`;158 await this.throttle();159 const res = await fetchPage(url);160 pages++;161 if (!res.success || !res.html) {162 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);163 break;164 }165 const payload = parseCataloguePage(res.html, auction, url);166 if (!payload.lots.length) {167 if (state.offset === 0) ctx.anomaly('parse_failure_page', `${url}: no lot cards`);168 state = { ...state, offset: -1 };169 break;170 }171 state = { auction, offset: state.offset, totalLots: payload.totalLots ?? state.totalLots };172 if (payload.lots.some((l) => l.hammerText && !/unsold/i.test(l.hammerText))) {173 yielded++;174 yield { url, externalId: `auction:${auction.id}:offset:${payload.offset}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };175 }176 const next = state.offset + PAGE_SIZE;177 const more = payload.lots.length >= PAGE_SIZE && (state.totalLots === null || next < state.totalLots);178 if (!more) {179 state = { ...state, offset: -1 };180 break;181 }182 state = { ...state, offset: next };183 cursor.inProgress = state;184 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-800), updatedAt: new Date().toISOString() });185 }186 if (state.offset === -1) {187 done.add(auction.id);188 finished++;189 cursor.inProgress = null;190 if (ctx.options.mode === 'backfill') {191 const idx = numismatic.findIndex((a) => a.id === auction.id);192 await ctx.progress({ page: idx + 1, totalPages: numismatic.length, itemsProcessed: yielded, reachedDate: parseAuctionDate(auction.dateText) });193 }194 } else cursor.inProgress = state;195 await ctx.setCursor({ ...cursor, doneAuctions: [...done].slice(-800), updatedAt: new Date().toISOString() });196 }197 if (ctx.options.mode === 'backfill' && numismatic.every((a) => done.has(a.id))) {198 await ctx.setCursor({ ...cursor, done: true, doneAuctions: [...done].slice(-800), updatedAt: new Date().toISOString() });199 }200 }201202 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {203 const p = PayloadSchema.parse(raw.payload);204 const saleDate = parseAuctionDate(p.auction.dateText);205 if (!saleDate) return [];206 const fallback = noonansDefaultSlug(p.auction.title) ?? 'coins';207 const out: NormalizedSale[] = [];208 for (const lot of p.lots) {209 const price = realized(lot.hammerText, 'GBP');210 if (!price) continue; // Unsold / withdrawn211 const categorySlug = numisCategory(lot.description, p.auction.title, fallback);212 const g = parseCoinGrade(lot.description);213 const est = lot.estimateText ? realized(lot.estimateText.split(/[-–]/)[0]!, price.currency) : null;214 const estHigh = lot.estimateText?.includes('-') || lot.estimateText?.includes('–') ? realized(lot.estimateText.split(/[-–]/).pop()!, price.currency) : null;215 const attributes = numisAttributes({216 categorySlug,217 title: lot.description,218 section: p.auction.title,219 identifiers: { noonans_lot: lot.lotId },220 metadata: { auction_id: p.auction.id, auction_title: p.auction.title, estimate_low: est?.amount ?? null, estimate_high: estHigh?.amount ?? est?.amount ?? null, estimate_currency: est?.currency ?? null, hammer_price: price.amount, buyer_premium: 'excluded (Noonans publishes hammer prices)' },221 });222 const sale = makeSale({223 meta: this.meta,224 sourceUrl: lot.url,225 externalId: lot.lotId,226 rawTitle: lot.description.length > 240 ? `${lot.description.slice(0, 239)}…` : lot.description,227 description: lot.description,228 attributes,229 price: price.amount,230 currency: price.currency,231 saleDate,232 buyerPremiumIncluded: false,233 auctionHouse: 'Noonans Mayfair',234 lotNumber: lot.lotNumber,235 imageUrls: lot.image ? [lot.image] : [],236 observedAt: raw.fetchedAt,237 parserVersion: PARSER_VERSION,238 confidence: g.grader ? 0.88 : 0.8,239 isBundle: isNumisBundle(lot.description),240 conditionRaw: g.conditionRaw,241 location: 'GB',242 });243 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: g.certificationNumber };244 out.push(sale);245 }246 return out;247 }248}249250export default (meta: ConnectorMeta) => new NoonansConnector(meta);251