TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';5import { dateWords, lotAttributes, makeSale, money } from '../../firecrawl/_carlib/index.js';67const BASE = 'https://www.swanngalleries.com';8const PARSER_VERSION = '1.0.0';9/** Swann answers 403 to bare product tokens; a Mozilla-compatible bot UA (still identifying RareIndex) is accepted. */10const UA = { 'user-agent': 'Mozilla/5.0 (compatible; RareIndexBot/0.1; +https://www.rareindex.io/about)' };1112export const AuctionSchema = z.object({ slug: z.string(), url: z.string(), title: z.string(), dateText: z.string().nullable(), department: z.string().nullable(), saleNumber: z.string().nullable() });13export const LotSchema = z.object({ ref: z.string(), url: z.string(), lotNumber: z.string().nullable(), title: z.string(), estimateText: z.string().nullable(), soldText: z.string().nullable(), passed: z.boolean(), premiumNote: z.boolean(), image: z.string().nullable() });14export const PagePayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });15export type PagePayload = z.infer<typeof PagePayloadSchema>;1617export function parsePastAuctions(htmlText: string): z.infer<typeof AuctionSchema>[] {18 const $ = H.load(htmlText);19 const out: z.infer<typeof AuctionSchema>[] = [];20 $('a.btn-cta--view_lots').each((_, a) => {21 const href = $(a).attr('href') ?? '';22 const m = href.match(/auction-catalog\/([^/?#]+)/);23 if (!m) return;24 // walk up to the nearest ancestor that carries the event title (markup nests the CTA deep inside the card)25 const card = $(a).parents().filter((_, el) => $(el).find('.event__title').length > 0).first();26 const scope = card.length ? card : $(a).parent();27 const title = H.text(scope.find('.event__title').first()) ?? H.text(scope.find('h2').first()) ?? '';28 const dateText = H.text(scope.find('.event__date').first());29 const dept = H.text(scope.find('.event__department a').first());30 const saleNumber = (H.text(scope.find('.event__department').first()) ?? '').match(/Sale\s+(\d+)/)?.[1] ?? null;31 if (title && !out.some((x) => x.slug === m[1])) out.push({ slug: m[1]!, url: href, title, dateText, department: dept, saleNumber });32 });33 return out;34}3536export function parseCatalogPage(htmlText: string, auction: z.infer<typeof AuctionSchema>, page: number): PagePayload {37 const $ = H.load(htmlText);38 const lots: z.infer<typeof LotSchema>[] = [];39 $('[data-lot-ref]').each((_, el) => {40 const $el = $(el);41 const ref = $el.attr('data-lot-ref') ?? '';42 const a = $el.find('a[href*="/auction-lot/"]').first();43 const url = a.attr('href') ?? '';44 const titleRaw = H.text($el.find('[class*="card-title"]').first()) ?? '';45 const tm = titleRaw.match(/^(\d+[A-Za-z]?):\s*(.*)$/s);46 const estimateText = $el.find('[class*="estimate-bid"] span').last().text().trim() || null;47 const amount = $el.find('[class*="bid-amount"] [class*="amount"] span').last().text().trim();48 const passed = /passed|unsold|withdrawn/i.test($el.text());49 if (!ref || !url || !titleRaw) return;50 lots.push({ ref, url, lotNumber: tm?.[1] ?? null, title: (tm?.[2] ?? titleRaw).trim(), estimateText, soldText: amount && /\d/.test(amount) ? amount : null, passed, premiumNote: /includes buyer/i.test($el.text()), image: $el.find('img').first().attr('src') ?? null });51 });52 return { kind: 'catalog_page', auction, page, lots };53}5455export function swannCategory(department: string | null, saleTitle: string, lotTitle: string): string {56 const d = `${department ?? ''} ${saleTitle}`.toLowerCase();57 const t = lotTitle.toLowerCase();58 if (/autograph/.test(d)) return /letter|document|manuscript|signed document|archive/.test(t) ? 'historical_documents' : 'autographs';59 if (/photograph/.test(d)) return 'photography';60 if (/poster/.test(d)) return 'movie_posters';61 if (/map|atlas/.test(d)) return 'maps';62 if (/illustration|animation|comic/.test(d)) return /cel\b|animation/.test(t) ? 'animation_art' : 'art';63 if (/contemporary|modern|african-american art|19th|20th|prints|drawings|art/.test(d)) return /contemporary|post-war/.test(d) ? 'contemporary_art' : 'art';64 if (/printed|manuscript|americana|books|literature|children|early printed/.test(d)) return /letter|manuscript|document|archive|autograph/.test(t) ? 'historical_documents' : 'books';65 return 'books';66}6768export class SwannConnector extends BaseConnector {69 readonly version = '1.0.0';70 readonly parserVersion = PARSER_VERSION;71 protected override minIntervalMs = 1500;7273 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {74 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);75 const pagesPerAuction = Number(this.meta.config.pagesPerAuction ?? 15);76 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);77 const listUrl = `${BASE}/auctions/past-auctions/`;78 await this.throttle();79 const list = await ctx.fetch(listUrl, { engines: ['api', 'firecrawl'], headers: UA, responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parsePastAuctions(r.html)[0]?.title ?? null, date: parsePastAuctions(r.html)[0]?.dateText ?? null } : null) });80 if (!list.success || !list.html) {81 ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);82 return;83 }84 const auctions = parsePastAuctions(list.html).filter((a) => !done.has(a.slug));85 let count = 0;86 let processed = 0;87 for (const auction of auctions) {88 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;89 for (let page = 1; page <= pagesPerAuction; page++) {90 if (ctx.signal?.aborted || this.reached(ctx, count)) break;91 const url = `${BASE}/auction-catalog/${auction.slug}?algoliaParam=${encodeURIComponent(`archive_lotNumber_asc_prod[page]=${page}`)}`;92 await this.throttle();93 const res = await ctx.fetch(url, {94 engines: ['api', 'firecrawl'],95 headers: UA,96 responseType: 'text',97 expect: ['title', 'price', 'status'],98 parse: (r) => {99 const p = r.html ? parseCatalogPage(r.html, auction, page) : null;100 const f = p?.lots.find((l) => l.soldText);101 return f ? { title: f.title, price: money(f.soldText, 'USD')?.amount ?? null, status: 'sold' } : p?.lots.length ? { title: p.lots[0]!.title } : null;102 },103 minQuality: 0.2,104 });105 if (!res.success || !res.html) {106 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);107 break;108 }109 const payload = parseCatalogPage(res.html, auction, page);110 if (payload.lots.length === 0) break;111 count++;112 yield { url, externalId: `catalog:${auction.slug}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };113 if (payload.lots.length < 20) break;114 }115 processed++;116 done.add(auction.slug);117 await ctx.setCursor({ doneAuctions: [...done].slice(-300), updatedAt: new Date().toISOString() });118 }119 }120121 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {122 const p = PagePayloadSchema.parse(raw.payload);123 const saleDate = dateWords(p.auction.dateText);124 if (!saleDate) return [];125 const out: NormalizedSale[] = [];126 for (const lot of p.lots) {127 if (!lot.soldText || lot.passed) continue;128 const m = money(lot.soldText, 'USD');129 if (!m) continue;130 const categorySlug = swannCategory(p.auction.department, p.auction.title, lot.title);131 const g = parseGradeFromTitle(lot.title);132 const year = lot.title.match(/\b(1[5-9]\d{2}|20\d{2})\b/)?.[1];133 const artist = lot.title.match(/^([A-Z][A-Za-z.'\- ]+?)(?:\.|,|\s\()/)?.[1] ?? null;134 const attributes = lotAttributes({ categorySlug, name: lot.title, brand: artist, year: year ? Number(year) : null, identifiers: { swann_lot_ref: lot.ref }, metadata: { sale_number: p.auction.saleNumber, sale_title: p.auction.title, department: p.auction.department, estimate: lot.estimateText } });135 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: lot.ref, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: lot.premiumNote ? true : null, auctionHouse: 'Swann Auction Galleries', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, location: 'US' }));136 }137 return out;138 }139}140141export default (meta: ConnectorMeta) => new SwannConnector(meta);142