TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { isBundleTitle, safeYear } from '../../api/_auction-lib/categories.js';5import { gradeOf, lotAttributes, money, selectedOption, sportsCategory } from '../../api/_memorabilia-lib/index.js';6import { dateMDY, makeSale } from '../_carlib/index.js';78const BASE = 'https://auction.lelands.com';9const PARSER_VERSION = '1.1.0';1011export const ItemSchema = z.object({ itemId: z.string(), title: z.string(), url: z.string(), image: z.string().nullable(), bids: z.number().nullable(), openingBid: z.number().nullable(), status: z.string().nullable(), soldPrice: z.number().nullable(), lotNumber: z.string().nullable() });12export const PayloadSchema = z.object({ kind: z.literal('gallery_page'), auction: z.object({ id: z.string().nullable(), name: z.string(), startText: z.string().nullable(), endText: z.string().nullable(), premiumIncluded: z.boolean() }), page: z.number(), totalPages: z.number().nullable(), items: z.array(ItemSchema) });13export type Payload = z.infer<typeof PayloadSchema>;1415export function parseGallery(htmlText: string, page: number): Payload {16 const $ = H.load(htmlText);17 const side = $('.sidebar-widget').first();18 const name = H.text(side.find('h5.title')) ?? '';19 const sideText = H.text(side) ?? '';20 const startText = sideText.match(/Start:\s*(\d{1,2}\/\d{1,2}\/\d{4}[^E]*E[SD]T)/)?.[1] ?? null;21 const endText = sideText.match(/End:\s*(\d{1,2}\/\d{1,2}\/\d{4}[^E]*E[SD]T)/)?.[1] ?? null;22 const premiumIncluded = /Prices Shown Include Buyer'?s Premium/i.test(sideText);23 const items: Payload['items'] = [];24 const seen = new Set<string>();25 $('.item').each((_, el) => {26 const it = $(el);27 const link = it.find('p.description a').first();28 const href = link.attr('href');29 const title = H.text(link);30 const itemId = href?.match(/itemid=(\d+)/i)?.[1];31 if (!href || !title || !itemId || seen.has(itemId)) return;32 seen.add(itemId);33 const meta = H.text(it.find('.item-details > p').last()) ?? '';34 const bids = meta.match(/Bids:\s*(\d+)/)?.[1];35 const opening = meta.match(/Opening Bid:\s*\$([\d,]+)/)?.[1];36 const status = meta.match(/Status:\s*([A-Za-z ]+)/)?.[1]?.trim() ?? null;37 const priceText = H.text(it.find('.item-price')) ?? '';38 const sold = priceText.match(/SOLD FOR\s*\$([\d,]+(?:\.\d+)?)/i)?.[1];39 items.push({ itemId, title, url: `${BASE}/bids/bidplace.aspx?itemid=${itemId}`, image: it.find('.item-image img').attr('src') ?? null, bids: bids ? Number(bids) : null, openingBid: opening ? money(`$${opening}`, 'USD')?.amount ?? null : null, status, soldPrice: sold ? money(`$${sold}`, 'USD')?.amount ?? null : null, lotNumber: H.text(it.find('h5.boxed')) });40 });41 const pages = $('ul.pagination a[href*="page="]')42 .map((_, a) => Number(($(a).attr('href') ?? '').match(/page=(\d+)/)?.[1] ?? 0))43 .get()44 .filter((n) => n > 0);45 return { kind: 'gallery_page', auction: { id: selectedOption(htmlText, 'ctl00$Auction'), name, startText, endText, premiumIncluded }, page, totalPages: pages.length ? Math.max(...pages) : null, items };46}4748/**49 * Lelands (sports memorabilia & cards). The public gallery auction.lelands.com/Lots/Gallery?size=250&page=N50 * lists the currently displayed auction's lots with "SOLD FOR $X"; the sidebar gives start/end and states that51 * prices include the buyer's premium. Direct requests receive a Cloudflare 403, so the pages are rendered52 * through Firecrawl (a browser render of the public page — no login, no bidding). Past auctions are only53 * reachable through a form postback, which is not attempted; each auction is captured while it is the54 * displayed one (lots stay marked SOLD after the close).55 */56export class LelandsConnector extends BaseConnector {57 readonly version = '1.1.0';58 readonly parserVersion = PARSER_VERSION;59 protected override minIntervalMs = 2500;6061 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {62 const maxPages = Number(this.meta.config.pagesPerRun ?? 5);63 const doneAuctions = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);64 let count = 0;65 let auctionId: string | null = null;66 for (let page = 1; page <= maxPages; page++) {67 if (ctx.signal?.aborted || this.reached(ctx, count)) break;68 const url = `${BASE}/Lots/Gallery?size=250${page > 1 ? `&page=${page}` : ''}`;69 await this.throttle();70 const res = await ctx.fetch(url, { engines: ['firecrawl'], expect: ['title', 'price', 'status', 'date'], parse: (r) => { const p = r.html ? parseGallery(r.html, page) : null; const f = p?.items.find((i) => i.soldPrice); return p ? { title: f?.title ?? p.items[0]?.title ?? null, price: f?.soldPrice ?? null, status: f?.status ?? null, date: p.auction.endText } : null; } });71 if (!res.success || !res.html) {72 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);73 break;74 }75 const payload = parseGallery(res.html, page);76 if (page === 1) {77 auctionId = payload.auction.id;78 if (!payload.auction.endText) ctx.anomaly('missing_auction_end', payload.auction.name);79 // Closed auctions are crawled once; the live one is refreshed each run.80 if (auctionId && doneAuctions.has(auctionId)) break;81 }82 if (payload.items.length === 0) break;83 count++;84 yield { url, externalId: `auction:${payload.auction.id ?? 'current'}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };85 if (payload.totalPages !== null && page >= payload.totalPages) {86 const end = dateMDY(payload.auction.endText);87 if (auctionId && end && end.getTime() < Date.now()) doneAuctions.add(auctionId);88 break;89 }90 }91 await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-100), updatedAt: new Date().toISOString() });92 }9394 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {95 const p = PayloadSchema.parse(raw.payload);96 const saleDate = dateMDY(p.auction.endText);97 const out: NormalizedRecord[] = [];98 if (!saleDate || saleDate.getTime() > Date.now()) return out; // auction still open → no realized prices yet99 for (const it of p.items) {100 if (it.soldPrice === null || it.soldPrice <= 0 || !/sold/i.test(it.status ?? '')) continue;101 const g = gradeOf(it.title);102 const attributes = lotAttributes({ categorySlug: sportsCategory(it.title), name: it.title, year: safeYear(it.title), identifiers: { lelands_item_id: it.itemId }, metadata: { auction_id: p.auction.id, auction_name: p.auction.name, bids: it.bids, opening_bid: it.openingBid } });103 out.push(makeSale({ meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, price: it.soldPrice, currency: 'USD', saleDate, buyerPremiumIncluded: p.auction.premiumIncluded ? true : null, auctionHouse: 'Lelands', lotNumber: it.lotNumber, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundleTitle(it.title) }));104 }105 return out;106 }107}108109export default (meta: ConnectorMeta) => new LelandsConnector(meta);110