spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateWords, lotAttributes, makeSale, money } from '../_carlib/index.js';5import { isBundleTitle, safeYear } from '../../api/_auction-lib/categories.js';67/**8 * Cherrystone Philatelic Auctioneers — prices realized per lot from the public catalog category pages.9 * Engine: Firecrawl (rawHtml). See meta.json accessNotes.10 */1112const SITE = 'https://auctions.cherrystoneauctions.com';13const PARSER_VERSION = '1.0.0';1415export const AuctionSchema = z.object({ id: z.string(), title: z.string(), dateText: z.string().nullable() });16export const LotSchema = z.object({ lotNumber: z.string(), title: z.string(), url: z.string(), image: z.string().nullable(), finalPriceText: z.string().nullable(), estimateText: z.string().nullable() });17export const PayloadSchema = z.object({ kind: z.literal('category_page'), auction: AuctionSchema, category: z.string(), categoryUrl: z.string(), lots: z.array(LotSchema) });18export type Payload = z.infer<typeof PayloadSchema>;1920function clean(s: string): string {21  return s.replace(/&amp;/g, '&').replace(/&#0?39;|&apos;/g, "'").replace(/&quot;/g, '"').replace(/&nbsp;/g, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();22}2324/** Home page (HTML or markdown) → closed sales with prices realized. */25export function parseHome(text: string): z.infer<typeof AuctionSchema>[] {26  const out: z.infer<typeof AuctionSchema>[] = [];27  const push = (id: string, title: string, dateText: string) => {28    if (!out.some((a) => a.id === id)) out.push({ id, title: clean(title), dateText });29  };30  const DATE = String.raw`([A-Za-z]+ \d{1,2}(?:\s*[-–]\s*\d{1,2})?,\s*\d{4})`;31  // HTML: <a href="…catalog.aspx?auctionid=34">Click here</a> for Prices Realized from <sale> - <date>32  const reHtml = new RegExp(String.raw`href="[^"]*catalog\.aspx\?auctionid=(\d+)"[^>]*>[^<]*<\/a>\s*for Prices Realized from\s+(.+?)\s+-\s+` + DATE, 'gi');33  let m: RegExpExecArray | null;34  while ((m = reHtml.exec(text))) push(m[1]!, m[2]!, m[3]!);35  // Markdown: [Click here](…catalog.aspx?auctionid=34) for Prices Realized from <sale> - <date>36  const reMd = new RegExp(String.raw`\]\([^)]*catalog\.aspx\?auctionid=(\d+)\)\s*for Prices Realized from\s+(.+?)\s+-\s+` + DATE, 'gi');37  while ((m = reMd.exec(text))) push(m[1]!, m[2]!, m[3]!);38  return out;39}4041/** Catalog page → category page URLs for this auction. */42export function parseCategoryLinks(html: string, auctionId: string): Array<{ name: string; url: string }> {43  const out: Array<{ name: string; url: string }> = [];44  const re = /href="((?:https:\/\/auctions\.cherrystoneauctions\.com)?\/Category\/([^"?]+)\.html\?auctionid=(\d+))"/g;45  let m: RegExpExecArray | null;46  while ((m = re.exec(html))) {47    if (m[3] !== auctionId) continue;48    const url = m[1]!.startsWith('http') ? m[1]! : `${SITE}${m[1]!}`;49    if (out.some((c) => c.url === url)) continue;50    const name = m[2]!.replace(/-\d+$/, '').replace(/_/g, ' ').trim();51    if (/^All$/i.test(name)) continue;52    out.push({ name, url });53  }54  return out;55}5657/** Category page → lots. */58export function parseCategoryPage(html: string, auction: z.infer<typeof AuctionSchema>, category: string, categoryUrl: string): Payload {59  const lots: z.infer<typeof LotSchema>[] = [];60  const blocks = html.split(/<div class="lot\s*">/).slice(1);61  for (const b of blocks) {62    const num = b.match(/id="LotNumber">([^<]+)</)?.[1]?.trim();63    const link = b.match(/id="LotName">\s*<a href="([^"]+)">([\s\S]*?)<\/a>/);64    if (!num || !link) continue;65    lots.push({66      lotNumber: num,67      title: clean(link[2]!),68      url: link[1]!,69      image: b.match(/<img class="lotImage" src="([^"]+)"/)?.[1] ?? null,70      finalPriceText: b.match(/Final Price:\s*([^<]+)</)?.[1]?.trim() ?? null,71      estimateText: b.match(/Estimate:\s*([^<]+)</)?.[1]?.trim() ?? null,72    });73  }74  return { kind: 'category_page', auction, category, categoryUrl, lots };75}7677export function cherrystoneCategory(title: string): string {78  const t = title.toLowerCase();79  if (/(banknote|bank note|currency|paper money|specimen note|\$\d+ note)/.test(t)) return 'banknotes';80  if (/\b(coin|medal|token|sovereign|ducat)\b/.test(t) && !/\bcover|stamp|postal/.test(t)) return 'coins';81  return 'stamps';82}8384export class CherrystoneConnector extends BaseConnector {85  readonly version = '1.0.0';86  readonly parserVersion = PARSER_VERSION;87  protected override minIntervalMs = 2000;8889  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {90    const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 1);91    const categoriesPerRun = Number(this.meta.config.categoriesPerRun ?? 25);92    const cursor = (ctx.options.cursor ?? {}) as { doneAuctions?: string[]; progress?: Record<string, string[]> };93    const done = new Set<string>(cursor.doneAuctions ?? []);94    const progress: Record<string, string[]> = cursor.progress ?? {};95    await this.throttle();96    const home = await ctx.fetch(`${SITE}/`, { expect: ['title', 'date'], parse: (r) => ({ title: parseHome(r.html ?? r.markdown ?? '')[0]?.title ?? null, date: parseHome(r.html ?? r.markdown ?? '')[0]?.dateText ?? null }) });97    if (!home.success) {98      ctx.anomaly('page_fetch_failed', `home: ${home.error ?? home.httpStatus}`);99      return;100    }101    const auctions = parseHome(home.html ?? home.markdown ?? '').filter((a) => !done.has(a.id));102    let count = 0;103    let processed = 0;104    for (const auction of auctions) {105      if (processed >= auctionsPerRun || ctx.signal?.aborted) break;106      await this.throttle();107      const cat = await ctx.fetch(`${SITE}/catalog.aspx?auctionid=${auction.id}`, { expect: ['title'], parse: (r) => ({ title: r.html ? parseCategoryLinks(r.html, auction.id)[0]?.name ?? null : null }) });108      if (!cat.success || !cat.html) {109        ctx.anomaly('page_fetch_failed', `catalog ${auction.id}: ${cat.error ?? cat.httpStatus}`);110        break;111      }112      const seen = new Set(progress[auction.id] ?? []);113      const categories = parseCategoryLinks(cat.html, auction.id).filter((c) => !seen.has(c.url));114      let n = 0;115      for (const c of categories) {116        if (n >= categoriesPerRun || ctx.signal?.aborted || this.reached(ctx, count)) break;117        await this.throttle();118        const res = await ctx.fetch(c.url, {119          expect: ['title', 'price'],120          parse: (r) => {121            const f = r.html ? parseCategoryPage(r.html, auction, c.name, c.url).lots.find((l) => l.finalPriceText) : null;122            return f ? { title: f.title, price: money(f.finalPriceText, 'USD')?.amount ?? null } : null;123          },124        });125        n++;126        seen.add(c.url);127        if (!res.success || !res.html) {128          ctx.anomaly('page_fetch_failed', `${c.url}: ${res.error ?? res.httpStatus}`);129          continue;130        }131        const payload = parseCategoryPage(res.html, auction, c.name, c.url);132        if (payload.lots.length === 0) continue;133        count++;134        yield { url: c.url, externalId: `auction:${auction.id}:${c.name}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };135      }136      progress[auction.id] = [...seen];137      const remaining = categories.length - n;138      if (remaining <= 0) {139        done.add(auction.id);140        delete progress[auction.id];141        processed++;142      }143      await ctx.setCursor({ doneAuctions: [...done].slice(-50), progress, updatedAt: new Date().toISOString() });144      if (remaining > 0) break; // continue this auction next run145    }146  }147148  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {149    const p = PayloadSchema.parse(raw.payload);150    const saleDate = dateWords(p.auction.dateText);151    if (!saleDate) return [];152    const out: NormalizedSale[] = [];153    for (const lot of p.lots) {154      const m = money(lot.finalPriceText, 'USD');155      if (!m) continue; // $0 / missing = unsold156      const categorySlug = cherrystoneCategory(lot.title);157      const attributes = lotAttributes({158        categorySlug,159        name: lot.title,160        set: p.category,161        year: safeYear(lot.title),162        identifiers: { cherrystone_lot: lot.url.match(/LOT(\d+)\.aspx/i)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` },163        metadata: { auction_id: p.auction.id, auction_title: p.auction.title, category: p.category, estimate: lot.estimateText, scott_catalogue: lot.title.match(/\(([0-9A-Za-z-]+)\)/)?.[1] ?? null },164      });165      out.push(166        makeSale({167          meta: this.meta,168          sourceUrl: lot.url,169          externalId: `${p.auction.id}-${lot.lotNumber}`,170          rawTitle: lot.title,171          attributes,172          price: m.amount,173          currency: 'USD',174          saleDate,175          buyerPremiumIncluded: null,176          auctionHouse: 'Cherrystone Auctions',177          lotNumber: lot.lotNumber,178          imageUrls: lot.image ? [lot.image] : [],179          observedAt: raw.fetchedAt,180          parserVersion: PARSER_VERSION,181          isBundle: isBundleTitle(lot.title) || /\b(collection|accumulation|balance|group of|lot of)\b/i.test(lot.title),182          location: 'US',183        }),184      );185    }186    return out;187  }188}189190export default (meta: ConnectorMeta) => new CherrystoneConnector(meta);191