TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { makeSale } from '../../firecrawl/_carlib/index.js';5import { clean, isNumisBundle, numisAttributes, numisCategory, parseAuctionDate, parseCoinGrade, realized } from '../../firecrawl/_g5-numismatics-lib/index.js';67/**8 * Noble Numismatics (Sydney) — every sale since 1994 publishes a single static "Prices Realised" page9 * listing all lots (description, grading text, price in AUD or "Passed in"). Plain HTTPS, no JS.10 * The page states "All prices exclude buyer and vendor premiums" → hammer prices.11 */1213const SITE = 'https://www.noble.com.au';14const PARSER_VERSION = '1.0.0';15const CHUNK = 250;1617export const SaleSchema = z.object({ number: z.string(), title: z.string(), dateText: z.string().nullable(), city: z.string().nullable() });18export const LotSchema = z.object({19 lotId: z.string(),20 lotNumber: z.string(),21 headline: z.string().nullable(),22 description: z.string(),23 gradingText: z.string().nullable(),24 priceText: z.string().nullable(),25 status: z.string().nullable(),26});27export const PayloadSchema = z.object({28 kind: z.literal('prices_realised'),29 sale: SaleSchema,30 url: z.string(),31 premiumNote: z.string().nullable(),32 chunk: z.number().int(),33 totalLots: z.number().int(),34 lots: z.array(LotSchema),35});36export type Payload = z.infer<typeof PayloadSchema>;3738export interface SaleIndexEntry extends z.infer<typeof SaleSchema> {39 hasPrices: boolean;40}4142/** /auctions → past sales (newest first). */43export function parseSalesIndex(htmlText: string): SaleIndexEntry[] {44 const $ = H.load(htmlText);45 const out: SaleIndexEntry[] = [];46 $('li.past-sale-row').each((_, li) => {47 const e = $(li);48 const number = e.attr('data-sale-number') ?? clean(e.find('.past-sale-row__label').text()).replace(/^Sale\s+/i, '');49 if (!number || out.some((s) => s.number === number)) return;50 const title = clean(e.find('.past-sale-row__title').first().text()) || `Sale ${number}`;51 const dateText = clean(e.find('.past-sale-row__date').first().text()) || null;52 const city = clean(e.find('.past-sale-row__city').first().text()) || null;53 const hasPrices = e.find(`a[href*="/auctions/sale/${number}/prices-realised"]`).length > 0;54 out.push({ number, title, dateText, city, hasPrices });55 });56 return out;57}5859export interface PricesRealisedPage {60 sale: z.infer<typeof SaleSchema>;61 premiumNote: string | null;62 lots: z.infer<typeof LotSchema>[];63}6465/** /auctions/sale/<n>/prices-realised → header + every lot row. */66export function parsePricesRealised(htmlText: string, number: string): PricesRealisedPage | null {67 const $ = H.load(htmlText);68 const title = clean($('.prices-realised__sale-title').first().text());69 if (!title) return null;70 const dateText = clean($('.prices-realised__dates').first().text()) || null;71 const city = clean($('.prices-realised__venue').first().text()) || null;72 const premiumNote = clean($('.prices-realised__total-note').first().text()) || null;73 const lots: z.infer<typeof LotSchema>[] = [];74 $('table.prices-realised__table tbody tr').each((_, tr) => {75 const row = $(tr);76 const a = row.find('td.prices-realised__cell-lot a').first();77 const lotId = a.attr('href')?.match(/[?&]id=(\d+)/)?.[1];78 const lotNumber = clean(a.text()).replace(/^Lot\s+/i, '');79 if (!lotId || !lotNumber) return;80 const desc = row.find('td.prices-realised__cell-desc').first();81 const headline = clean(desc.find('strong').first().text()).replace(/,\s*$/, '') || null;82 const gradingText = clean(desc.find('em').first().text()) || null;83 const description = clean(desc.html() ?? '');84 const priceCell = row.find('td.prices-realised__cell-price').first();85 const status = clean(priceCell.find('.prices-realised__status').first().text()) || null;86 const priceText = status ? null : clean(priceCell.text()) || null;87 lots.push({ lotId, lotNumber, headline, description, gradingText, priceText, status });88 });89 return { sale: { number, title, dateText, city }, premiumNote, lots };90}9192interface Cursor {93 doneSales?: string[];94 done?: boolean;95 updatedAt?: string;96}9798export class NobleNumismaticsConnector extends BaseConnector {99 readonly version = '1.0.0';100 readonly parserVersion = PARSER_VERSION;101 protected override minIntervalMs = 2000;102 override readonly urlPatterns = [/noble\.com\.au\/auctions\/(?:sale|lot)/i];103104 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {105 const salesPerRun = Number(this.meta.config.salesPerRun ?? 1);106 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };107 const done = new Set(cursor.doneSales ?? []);108 await this.throttle();109 const index = await ctx.fetch(`${SITE}/auctions`, { engines: ['api'], responseType: 'text', expect: ['title', 'date'], parse: (r) => ({ title: r.html ? parseSalesIndex(r.html)[0]?.title ?? null : null, date: r.html ? parseSalesIndex(r.html)[0]?.dateText ?? null : null }) });110 if (!index.success || !index.html) {111 ctx.anomaly('page_fetch_failed', `/auctions: ${index.error ?? index.httpStatus}`);112 return;113 }114 const sales = parseSalesIndex(index.html).filter((s) => s.hasPrices);115 if (!sales.length) {116 ctx.anomaly('selector_missing', '/auctions: no past-sale rows with prices realised');117 return;118 }119 const queue = sales.filter((s) => !done.has(s.number));120 let processed = 0;121 let yielded = 0;122 for (const sale of queue) {123 if (ctx.signal?.aborted || processed >= salesPerRun || this.reached(ctx, yielded)) break;124 const url = `${SITE}/auctions/sale/${sale.number}/prices-realised`;125 await this.throttle();126 const res = await ctx.fetch(url, {127 engines: ['api'],128 responseType: 'text',129 expect: ['title', 'price'],130 parse: (r) => {131 const p = r.html ? parsePricesRealised(r.html, sale.number) : null;132 const sold = p?.lots.find((l) => l.priceText);133 return p ? { title: p.sale.title, price: sold?.priceText ?? null } : null;134 },135 });136 processed++;137 if (!res.success || !res.html) {138 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);139 continue;140 }141 const page = parsePricesRealised(res.html, sale.number);142 if (!page || !page.lots.length) {143 ctx.anomaly('parse_failure_page', `${url}: no lot rows`);144 continue;145 }146 // the index carries the sale date when the prices page omits it (rare)147 const saleInfo = { ...page.sale, dateText: page.sale.dateText ?? sale.dateText, city: page.sale.city ?? sale.city };148 for (let i = 0; i < page.lots.length; i += CHUNK) {149 if (ctx.signal?.aborted || this.reached(ctx, yielded)) break;150 const lots = page.lots.slice(i, i + CHUNK);151 if (!lots.some((l) => l.priceText)) continue;152 yielded++;153 const payload: Payload = { kind: 'prices_realised', sale: saleInfo, url, premiumNote: page.premiumNote, chunk: i / CHUNK, totalLots: page.lots.length, lots };154 yield { url, externalId: `sale:${sale.number}:chunk:${i / CHUNK}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };155 }156 done.add(sale.number);157 if (ctx.options.mode === 'backfill') {158 const idx = sales.findIndex((s) => s.number === sale.number);159 await ctx.progress({ page: idx + 1, totalPages: sales.length, itemsProcessed: yielded, reachedDate: parseAuctionDate(saleInfo.dateText) });160 }161 await ctx.setCursor({ doneSales: [...done].slice(-500), updatedAt: new Date().toISOString() });162 }163 if (ctx.options.mode === 'backfill' && sales.every((s) => done.has(s.number))) await ctx.setCursor({ doneSales: [...done].slice(-500), done: true, updatedAt: new Date().toISOString() });164 }165166 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {167 const p = PayloadSchema.parse(raw.payload);168 const saleDate = parseAuctionDate(p.sale.dateText);169 if (!saleDate) return [];170 const premiumExcluded = p.premiumNote ? /exclude.*premium/i.test(p.premiumNote) : null;171 const out: NormalizedSale[] = [];172 for (const lot of p.lots) {173 const price = realized(lot.priceText, 'AUD');174 if (!price) continue; // Passed in / withdrawn175 const categorySlug = numisCategory(lot.description, p.sale.title, 'coins');176 const g = parseCoinGrade(lot.description);177 const attributes = numisAttributes({178 categorySlug,179 title: lot.description,180 section: p.sale.title,181 country: undefined,182 identifiers: { noble_lot: lot.lotId },183 metadata: { sale_number: p.sale.number, sale_title: p.sale.title, city: p.sale.city, headline: lot.headline, grading_text: lot.gradingText, hammer_price: price.amount, buyer_premium: p.premiumNote ?? 'not stated' },184 });185 const sale = makeSale({186 meta: this.meta,187 sourceUrl: `${SITE}/auctions/lot/?id=${lot.lotId}`,188 externalId: lot.lotId,189 rawTitle: lot.description.length > 240 ? `${lot.description.slice(0, 239)}…` : lot.description,190 description: lot.description,191 attributes,192 price: price.amount,193 currency: 'AUD',194 saleDate,195 buyerPremiumIncluded: premiumExcluded === null ? null : !premiumExcluded,196 auctionHouse: 'Noble Numismatics',197 lotNumber: lot.lotNumber,198 observedAt: raw.fetchedAt,199 parserVersion: PARSER_VERSION,200 confidence: g.grader ? 0.85 : 0.78,201 isBundle: isNumisBundle(lot.description),202 conditionRaw: lot.gradingText?.replace(/\.\s*$/, '') ?? g.conditionRaw,203 location: 'AU',204 });205 sale.grade = { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: g.certificationNumber };206 out.push(sale);207 }208 return out;209 }210}211212export default (meta: ConnectorMeta) => new NobleNumismaticsConnector(meta);213