TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';5import { gradeOf, hiddenFields, httpText, lotAttributes, money, morphyCategory, selectedOption } from '../_memorabilia-lib/index.js';6import { dateWords, makeSale } from '../../firecrawl/_carlib/index.js';78const WP = 'https://morphyauctions.com';9const CAT = 'https://auctions.morphyauctions.com';10const PARSER_VERSION = '1.0.0';1112export const AuctionSchema = z.object({ id: z.string(), title: z.string(), dept: z.string().nullable(), dateText: z.string().nullable(), pageUrl: z.string().nullable() });13export const LotSchema = z.object({ lotNumber: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), finalPrice: z.number().nullable(), minBid: z.number().nullable(), estimateText: z.string().nullable(), bids: z.number().nullable() });14export const PayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) });15export type Payload = z.infer<typeof PayloadSchema>;1617/** WordPress past-auction list: department label, title, date and the catalog id. */18export function parsePastList(htmlText: string): z.infer<typeof AuctionSchema>[] {19 const $ = H.load(htmlText);20 const out: z.infer<typeof AuctionSchema>[] = [];21 $('article.event').each((_, el) => {22 const a = $(el);23 const link = a.find('a[href*="catalog.aspx?auctionid="]').attr('href');24 const id = link?.match(/auctionid=(\d+)/)?.[1];25 if (!id) return;26 out.push({ id, title: H.text(a.find('h3')) ?? '', dept: H.text(a.find('.category-title')), dateText: H.text(a.find('.date')), pageUrl: a.find('h3 a').attr('href') ?? null });27 });28 return out;29}3031export function parseCatalog(htmlText: string, auction: z.infer<typeof AuctionSchema>, page: number): Payload {32 const $ = H.load(htmlText);33 const lots: z.infer<typeof LotSchema>[] = [];34 $('#galleryList li .lot, #galleryList .lot').each((_, el) => {35 const l = $(el);36 const link = l.find('#LotName a, span[id="LotName"] a').first();37 const title = H.text(link);38 const href = link.attr('href');39 if (!title || !href) return;40 const data = H.text(l.find('.lotData')) ?? '';41 const fp = data.match(/Final Price:\s*\$([\d,]+(?:\.\d+)?)/i);42 const mb = data.match(/Min Bid:\s*\$([\d,]+(?:\.\d+)?)/i);43 const bids = data.match(/#\s*Bids:\s*(\d+)/i);44 const est = data.match(/Estimate:\s*([^\n]+?)(?:\s*$|\s*#|\s*Min|\s*Final)/i);45 lots.push({46 lotNumber: H.text(l.find('#LotNumber, span[id="LotNumber"]')),47 title,48 url: href.replace(/^http:/, 'https:'),49 image: (() => {50 const src = l.find('img.lotImage').attr('src');51 return src ? (src.startsWith('http') ? src : CAT + src) : null;52 })(),53 finalPrice: fp ? money(`$${fp[1]}`, 'USD')?.amount ?? null : null,54 minBid: mb ? money(`$${mb[1]}`, 'USD')?.amount ?? null : null,55 estimateText: est ? est[1]!.trim() : null,56 bids: bids ? Number(bids[1]) : null,57 });58 });59 const totalPages = Number(htmlText.match(/id\s*=\s*"ofpages">\s*\/\s*(\d+)/)?.[1] ?? '') || null;60 return { kind: 'catalog_page', auction, page, totalPages, lots };61}6263/**64 * Morphy Auctions: past-auction list on morphyauctions.com (WordPress, /page/N/) → auction catalogs on65 * auctions.morphyauctions.com (ASP.NET). Page 1 is a GET; later pages are the page's own form POST66 * (page number + "Go" button with the hidden __VIEWSTATE fields) — no login involved.67 */68export class MorphyConnector extends BaseConnector {69 readonly version = '1.0.0';70 readonly parserVersion = PARSER_VERSION;71 protected override minIntervalMs = 2000;7273 private stat(ctx: CrawlContext, ok: boolean, ms: number) {74 const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });75 s.attempts++;76 if (ok) s.success++;77 s.ms += ms;78 }7980 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {81 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);82 const maxPages = Number(this.meta.config.pagesPerAuction ?? 10);83 const perPage = String(this.meta.config.lotsPerPage ?? 100);84 const backfill = ctx.options.mode === 'backfill';85 const listPage = backfill ? Number(ctx.options.cursor?.listPage ?? 1) : 1;86 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);87 const listUrl = `${WP}/auctions/past-auctions/${listPage > 1 ? `page/${listPage}/` : ''}`;88 await this.throttle();89 const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3 });90 if (!list.success || !list.html) {91 ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);92 return;93 }94 const auctions = parsePastList(list.html).filter((a) => !done.has(a.id) && morphyCategory(a.dept, a.title, '') !== null);95 let processed = 0;96 let count = 0;97 for (const auction of auctions) {98 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;99 const url = `${CAT}/catalog.aspx?auctionid=${auction.id}`;100 await this.throttle();101 const t0 = Date.now();102 let res;103 try {104 res = await httpText(url);105 } catch (err) {106 this.stat(ctx, false, Date.now() - t0);107 ctx.anomaly('page_fetch_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);108 continue;109 }110 this.stat(ctx, res.status < 400, Date.now() - t0);111 if (res.status >= 400) {112 ctx.anomaly('page_fetch_failed', `${url}: HTTP ${res.status}`);113 continue;114 }115 let html = res.text;116 let payload = parseCatalog(html, auction, 1);117 let page = 1;118 // Switch to the larger page size through the form (what the page-size dropdown does).119 if (perPage !== '25' && payload.lots.length > 0) {120 const posted = await this.postPage(url, html, 1, perPage, ctx, 'ctl00$ContentPlaceHolder$LotsPerPageDropDownTop');121 if (posted) {122 html = posted;123 payload = parseCatalog(html, auction, 1);124 }125 }126 while (true) {127 if (payload.lots.length === 0) break;128 count++;129 yield { url: `${url}&page=${page}`, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };130 page++;131 if (this.reached(ctx, count) || page > maxPages || (payload.totalPages !== null && page > payload.totalPages) || ctx.signal?.aborted) break;132 await this.throttle();133 const next = await this.postPage(url, html, page, perPage, ctx);134 if (!next) break;135 html = next;136 payload = parseCatalog(html, auction, page);137 }138 processed++;139 done.add(auction.id);140 await ctx.setCursor({ doneAuctions: [...done].slice(-400), listPage: backfill && auctions.every((a) => done.has(a.id)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() });141 }142 }143144 /** Jump to `page` with `perPage` lots by posting the catalog form (ASP.NET postback). */145 private async postPage(url: string, currentHtml: string, page: number, perPage: string, ctx: CrawlContext, eventTarget?: string): Promise<string | null> {146 const hidden = hiddenFields(currentHtml);147 const form: Record<string, string> = {};148 for (const [k, v] of Object.entries(hidden)) if (k.startsWith('__') || k.startsWith('categoryView')) form[k] = v;149 const keep = ['ctl00$ContentPlaceHolder$SortByDDLTop', 'ctl00$ContentPlaceHolder$SortByDDLBot', 'ctl00$ContentPlaceHolder$displayByDropDownTop', 'ctl00$ContentPlaceHolder$displayByDropDownBot', 'ctl00$ContentPlaceHolder$searchByDropDown'];150 for (const k of keep) {151 const v = selectedOption(currentHtml, k);152 if (v !== null) form[k] = v;153 }154 // The auction dropdown defaults to the live sale, not the one being viewed: post the requested id explicitly.155 form['ctl00$ContentPlaceHolder$AuctionDDL'] = url.match(/auctionid=(\d+)/)?.[1] ?? '';156 form['ctl00$ContentPlaceHolder$LotsPerPageDropDownTop'] = perPage;157 form['ctl00$ContentPlaceHolder$LotsPerPageDropDownBot'] = perPage;158 form['ctl00$ContentPlaceHolder$CurrPageTopTB'] = String(page);159 form['ctl00$ContentPlaceHolder$CurrPageBotTB'] = String(page);160 form['ctl00$ContentPlaceHolder$searchTextBox'] = '';161 if (eventTarget) form.__EVENTTARGET = eventTarget;162 else form['ctl00$ContentPlaceHolder$PageJumpBtn'] = 'Go';163 const t0 = Date.now();164 try {165 const res = await httpText(url, { method: 'POST', form, referer: url });166 this.stat(ctx, res.status < 400, Date.now() - t0);167 if (res.status >= 400) return null;168 const cur = Number(res.text.match(/value="(\d+)" id="CurrPageTopTB"/)?.[1] ?? '0');169 if (cur !== page) {170 ctx.anomaly('pagination_mismatch', `${url}: asked page ${page}, got ${cur}`);171 return null;172 }173 return res.text;174 } catch (err) {175 this.stat(ctx, false, Date.now() - t0);176 ctx.anomaly('page_fetch_failed', `${url} (postback): ${err instanceof Error ? err.message : String(err)}`);177 return null;178 }179 }180181 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {182 const p = PayloadSchema.parse(raw.payload);183 const saleDate = dateWords(p.auction.dateText);184 const out: NormalizedRecord[] = [];185 if (!saleDate) return out;186 for (const lot of p.lots) {187 if (lot.finalPrice === null || lot.finalPrice <= 0) continue;188 const slug = morphyCategory(p.auction.dept, p.auction.title, lot.title);189 if (!slug) continue;190 const g = gradeOf(lot.title);191 const attributes = lotAttributes({ categorySlug: slug, name: lot.title, year: safeYear(lot.title), identifiers: { morphy_lot: lot.url.match(/LOT(\d+)\.aspx/i)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, department: p.auction.dept, estimate: lot.estimateText, min_bid: lot.minBid, bids: lot.bids } });192 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber ?? lot.url}`, rawTitle: lot.title, attributes, price: lot.finalPrice, currency: 'USD', saleDate, buyerPremiumIncluded: null, auctionHouse: 'Morphy Auctions', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundleTitle(lot.title) }));193 }194 return out;195 }196}197198export default (meta: ConnectorMeta) => new MorphyConnector(meta);199