TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { html as H, type ConnectorMeta } from '@rareindex/connectors';2import { parseGradeFromTitle } from '@rareindex/taxonomy';3import type { AssetAttributes, CurrencyCode, NormalizedAuctionLot, NormalizedCatalogItem, NormalizedListing } from '@rareindex/shared';4import { lotAttributes, money } from '../../firecrawl/_carlib/index.js';56/**7 * Helpers shared by the memorabilia / toys / dealer connectors (SCP, Hake's, Morphy, Lelands, BBTS,8 * Noble Knight, Trainz). Source vocabulary stays here; canonical models stay clean.9 */1011export const BOT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) RareIndexBot/0.1 (+https://www.rareindex.io/about; market-data research)';1213// ---------------------------------------------------------------------------------------------14// Plain HTTP helpers for sites whose paging/selection is an ASP.NET postback (what a browser does:15// a form POST carrying the page's own hidden fields; no authentication involved).16// ---------------------------------------------------------------------------------------------17export interface HttpResult {18 status: number;19 text: string;20 cookies: string[];21}2223export async function httpText(url: string, opts: { method?: 'GET' | 'POST'; form?: Record<string, string>; cookie?: string; referer?: string; timeoutMs?: number } = {}): Promise<HttpResult> {24 const ctrl = new AbortController();25 const timer = setTimeout(() => ctrl.abort(), opts.timeoutMs ?? 45_000);26 try {27 const headers: Record<string, string> = { 'user-agent': BOT_UA, accept: 'text/html,application/xhtml+xml,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.9' };28 if (opts.cookie) headers.cookie = opts.cookie;29 if (opts.referer) headers.referer = opts.referer;30 let body: string | undefined;31 if (opts.method === 'POST') {32 headers['content-type'] = 'application/x-www-form-urlencoded';33 body = new URLSearchParams(opts.form ?? {}).toString();34 }35 const res = await fetch(url, { method: opts.method ?? 'GET', headers, body, signal: ctrl.signal, redirect: 'follow' });36 const text = await res.text();37 const cookies: string[] = [];38 const anyHeaders = res.headers as Headers & { getSetCookie?: () => string[] };39 if (typeof anyHeaders.getSetCookie === 'function') cookies.push(...anyHeaders.getSetCookie());40 else {41 const sc = res.headers.get('set-cookie');42 if (sc) cookies.push(sc);43 }44 return { status: res.status, text, cookies };45 } finally {46 clearTimeout(timer);47 }48}4950/** Cookie header from Set-Cookie lines (name=value only). */51export function cookieHeader(setCookies: string[], previous = ''): string {52 const jar = new Map<string, string>();53 for (const part of previous.split(';')) {54 const [k, ...v] = part.trim().split('=');55 if (k) jar.set(k, v.join('='));56 }57 for (const sc of setCookies) {58 const first = sc.split(';')[0] ?? '';59 const [k, ...v] = first.trim().split('=');60 if (k) jar.set(k, v.join('='));61 }62 return [...jar.entries()].map(([k, v]) => `${k}=${v}`).join('; ');63}6465/** ASP.NET hidden inputs (__VIEWSTATE, __EVENTVALIDATION, …) plus any other hidden fields. */66export function hiddenFields(htmlText: string): Record<string, string> {67 const out: Record<string, string> = {};68 const re = /<input[^>]+type="hidden"[^>]*>/gi;69 let m: RegExpExecArray | null;70 while ((m = re.exec(htmlText))) {71 const tag = m[0];72 const name = tag.match(/name="([^"]+)"/)?.[1];73 if (!name) continue;74 const value = tag.match(/value="([^"]*)"/)?.[1] ?? '';75 out[name] = value.replace(/"/g, '"').replace(/&/g, '&');76 }77 return out;78}7980/** Selected value of a <select name="…"> (first option when none is marked). */81export function selectedOption(htmlText: string, name: string): string | null {82 const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));83 if (!m) return null;84 const sel = m[1]!.match(/<option[^>]*selected(?:="selected")?[^>]*value="([^"]*)"/i) ?? m[1]!.match(/<option[^>]*value="([^"]*)"[^>]*selected/i);85 if (sel) return sel[1]!;86 return m[1]!.match(/<option[^>]*value="([^"]*)"/i)?.[1] ?? null;87}8889export function selectOptions(htmlText: string, name: string): Array<{ value: string; label: string }> {90 const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));91 if (!m) return [];92 const out: Array<{ value: string; label: string }> = [];93 const re = /<option[^>]*value="([^"]*)"[^>]*>([^<]*)</gi;94 let o: RegExpExecArray | null;95 while ((o = re.exec(m[1]!))) out.push({ value: o[1]!, label: o[2]!.replace(/&/g, '&').trim() });96 return out;97}9899// ---------------------------------------------------------------------------------------------100// Bidsquare-hosted catalog sites (SCP Auctions, Hake's) share one HTML structure.101// ---------------------------------------------------------------------------------------------102export interface BidsquareEvent {103 id: string;104 name: string;105 url: string;106 status: 'upcoming' | 'live' | 'past' | 'unknown';107 startDate: string | null;108 endDate: string | null;109}110export interface BidsquareItem {111 itemId: string;112 url: string;113 title: string;114 lotNumber: string | null;115 image: string | null;116 /** "Sold for" | "Current Bid" | "Starting Bid" | "Passed" | null */117 priceLabel: string | null;118 price: number | null;119 bids: number | null;120 estimateLow: number | null;121 estimateHigh: number | null;122 /** unix seconds from the countdown script (start, end) when present */123 startsAt: number | null;124 endsAt: number | null;125}126export interface BidsquareCatalog {127 event: BidsquareEvent;128 page: number;129 totalPages: number | null;130 items: BidsquareItem[];131}132133export function parseBidsquareEvents(htmlText: string): BidsquareEvent[] {134 const $ = H.load(htmlText);135 const out: BidsquareEvent[] = [];136 $('.gtm-visible_event').each((_, el) => {137 const e = $(el);138 const id = e.attr('data-event_id');139 const name = e.attr('data-event_name') ?? H.text(e.find('h1')) ?? '';140 const href = e.find('h1 a').attr('href') ?? e.find('a[href*="/auctions/"]').first().attr('href');141 if (!id || !href) return;142 const st = e.attr('data-event_status') ?? 'unknown';143 out.push({ id, name, url: href.split('#')[0]!, status: st === 'past' || st === 'upcoming' || st === 'live' ? st : 'unknown', startDate: null, endDate: null });144 });145 return out;146}147148export function parseBidsquareCatalog(htmlText: string, page: number): BidsquareCatalog | null {149 const $ = H.load(htmlText);150 let event: BidsquareEvent | null = null;151 for (const ld of H.jsonLd(htmlText, 'Event')) {152 const url = String(ld.url ?? '');153 const id = url.match(/-(\d+)$/)?.[1] ?? null;154 if (!id) continue;155 event = { id, name: String(ld.name ?? ''), url, status: 'unknown', startDate: ld.startDate ? String(ld.startDate) : null, endDate: ld.endDate ? String(ld.endDate) : null };156 break;157 }158 const first = $('[data-item_id][data-event_id]').first();159 if (!event) {160 const id = first.attr('data-event_id');161 if (!id) return null;162 event = { id, name: first.attr('data-event_name') ?? '', url: '', status: 'unknown', startDate: null, endDate: null };163 }164 const st = first.attr('data-event_status');165 if (st === 'past' || st === 'upcoming' || st === 'live') event.status = st;166 const items: BidsquareItem[] = [];167 const seen = new Set<string>();168 $('div[id^="stl-"][data-item_id]').each((_, el) => {169 const c = $(el);170 const itemId = c.attr('data-item_id')!;171 if (seen.has(itemId)) return;172 seen.add(itemId);173 const link = c.find('.lot_title a').first();174 const url = link.attr('href') ?? c.find('.catalog_img a').attr('href') ?? '';175 const title = H.text(link) ?? c.find('.catalog_img img').attr('alt') ?? '';176 if (!url || !title) return;177 const lotNumber = H.text(c.find('.lot_Num'))?.replace(/^Lot\s*/i, '') ?? null;178 const image = c.find('.catalog_img img').attr('src') ?? null;179 const label = H.text(c.find('.bid_txt span').first());180 const priceText = H.text(c.find('.bid_txt .price').first());181 const bidsText = H.text(c.find('.bidLength .num').first());182 const est = c.find('.estimate_amount').attr('data-exchange');183 let estimateLow: number | null = null;184 let estimateHigh: number | null = null;185 if (est) {186 try {187 const j = JSON.parse(est.replace(/"/g, '"')) as { low_est?: string; high_est?: string };188 estimateLow = j.low_est ? Number(j.low_est) : null;189 estimateHigh = j.high_est ? Number(j.high_est) : null;190 } catch {191 /* ignore */192 }193 }194 const script = c.find('script').text();195 const cd = script.match(/countDownTimer\((\d+),\s*(\d+)/);196 items.push({197 itemId,198 url,199 title,200 lotNumber,201 image,202 priceLabel: label ?? null,203 price: money(priceText, 'USD')?.amount ?? null,204 bids: bidsText ? Number(bidsText.replace(/[^\d]/g, '')) || null : null,205 estimateLow,206 estimateHigh,207 startsAt: cd ? Number(cd[1]) : null,208 endsAt: cd ? Number(cd[2]) : null,209 });210 });211 const pages = $('ul.pagination li[data-page]')212 .map((_, li) => Number($(li).attr('data-page')))213 .get()214 .filter((n) => Number.isFinite(n) && n > 0);215 return { event, page, totalPages: pages.length ? Math.max(...pages) : null, items };216}217218/** "2026-07-26 22:30:00 EDT" → Date (Bidsquare prints US Eastern local time). */219export function bidsquareDate(s: string | null | undefined): Date | null {220 if (!s) return null;221 const m = s.match(/(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})\s*(EDT|EST|PDT|PST|CDT|CST|MDT|MST|UTC)?/);222 if (!m) return null;223 const offsets: Record<string, number> = { EDT: 4, EST: 5, CDT: 5, CST: 6, MDT: 6, MST: 7, PDT: 7, PST: 8, UTC: 0 };224 const off = offsets[m[7] ?? 'EDT'] ?? 4;225 return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]) + off, Number(m[5]), Number(m[6])));226}227228// ---------------------------------------------------------------------------------------------229// Category mappers (keyword based; null/family fallbacks, never invented attributes)230// ---------------------------------------------------------------------------------------------231const SPORT_WORDS: Array<[RegExp, string]> = [232 [/\b(baseball|topps|bowman|goudey|t206|bond bread|play ball|leaf 1948|mlb|world series|yankees|red sox|dodgers|jackie robinson|babe ruth|mickey mantle|lou gehrig|ty cobb|honus wagner)\b/i, 'baseball'],233 [/\b(basketball|nba|fleer basketball|fleer|hoops|jordan rookie|michael jordan|lebron|kobe|lakers|celtics)\b/i, 'basketball'],234 [/\b(football|nfl|super bowl|quarterback|packers|cowboys|steelers)\b/i, 'football'],235 [/\b(hockey|nhl|stanley cup|o-pee-chee|opc|canadiens|bruins|maple leafs)\b/i, 'hockey'],236 [/\b(soccer|fifa|world cup|premier league|messi|ronaldo|pel[eé])\b/i, 'soccer'],237];238const CARD_WORDS = /\b(card|rookie|psa|sgc|bgs|beckett|cgc|wax pack|unopened|refractor|parallel|#\s?\d+|1\/1)\b/i;239const MEMORABILIA_WORDS = /\b(jersey|game[- ]used|game[- ]worn|bat\b|glove|helmet|ball\b|puck|ring\b|trophy|ticket|program|contract|photo|photograph|signed|autograph|cleats|shoes|sneakers|medal|pennant|uniform)\b/i;240241/** Sports houses (SCP, Lelands, Hunt): cards by sport vs memorabilia. */242export function sportsCategory(title: string): string {243 const isCard = CARD_WORDS.test(title) && !MEMORABILIA_WORDS.test(title);244 if (isCard) {245 for (const [re, s] of SPORT_WORDS) if (re.test(title)) return `${s}_cards`;246 if (/\b(pok[eé]mon|charizard)\b/i.test(title)) return 'pokemon';247 if (/\b(magic: the gathering|mtg|black lotus)\b/i.test(title)) return 'magic_the_gathering';248 return 'other_sports_cards';249 }250 if (/\b(olympic|olympics)\b/i.test(title)) return 'olympic_collectibles';251 if (/\b(boxing|ufc|golf|tennis|nascar|wrestling|wwe|wwf|racing)\b/i.test(title)) return 'sports_memorabilia';252 return 'sports_memorabilia';253}254255/** Pop-culture houses (Hake's): toys, comics, political, pins… */256export function popCultureCategory(title: string): string {257 const t = title;258 if (/\bcgc\b|\bcbcs\b|comic book|#\s?\d+\s*\(?(19|20)\d\d|amazing spider-man|detective comics|action comics|x-men|batman|superman|marvel comics|dc comics/i.test(t) && !/\b(afa|figure|playset|vehicle)\b/i.test(t)) {259 if (/\b(marvel|spider-man|x-men|avengers|hulk|fantastic four|iron man|captain america|thor)\b/i.test(t)) return 'marvel_comics';260 if (/\b(dc|batman|superman|detective comics|action comics|wonder woman|flash|green lantern)\b/i.test(t)) return 'dc_comics';261 return 'independent_comics';262 }263 if (/\b(original art|comic art|cover art|splash page|comic book page|storyboard)\b/i.test(t)) return 'animation_art';264 if (/\b(campaign|president|political|button\b.*\b(19|18)\d\d|inaugur|senator|governor|election)\b/i.test(t)) return 'political_memorabilia';265 if (/\b(star wars|kenner|luke skywalker|darth vader|boba fett|tusken|stormtrooper|jedi|sith|millennium falcon)\b/i.test(t)) return 'star_wars';266 if (/\b(garbage pail kids|gpk|wacky packages|non-sport|trading card set)\b/i.test(t)) return 'non_sport_cards';267 if (/\b(psa|sgc|bgs)\s*\d|\btopps\b|\bbowman\b|rookie card/i.test(t)) return sportsCategory(t);268 if (/\b(pinback|pin-back|pin back|\bpin\b|badge)\b/i.test(t)) return 'pins';269 if (/\b(disney|mickey mouse|donald duck|disneyland|walt disney)\b/i.test(t)) return 'disney_collectibles';270 if (/\b(lunch box|lunchbox|thermos|pez|board game|premium ring|cereal|advertising|sign\b)\b/i.test(t)) return 'advertising';271 if (/\b(animation cel|production cel|cel\b)\b/i.test(t)) return 'animation_art';272 if (/\b(record|album|lp\b|concert|beatles|elvis|kiss\b)\b/i.test(t)) return 'music_memorabilia';273 if (/\b(movie|film|poster|lobby card|one sheet|screen[- ]used|prop\b)\b/i.test(t)) return 'movie_posters';274 if (/\b(signed|autograph)\b/i.test(t)) return 'autographs';275 if (/\b(g\.?i\.? joe|transformers|he-man|masters of the universe|tmnt|teenage mutant|action figure|afa\b|moc\b|mib\b|playset|hot wheels|matchbox|tin toy|wind-up|battery op|marx|ideal|mego|mattel|hasbro|lionel)\b/i.test(t)) return 'vintage_toys';276 if (/\b(jersey|game used|game worn|baseball|football|basketball|hockey|boxing)\b/i.test(t)) return 'sports_memorabilia';277 return 'vintage_toys';278}279280/** Morphy: department label + auction/lot titles → slug; firearms lots are excluded upstream. */281export function morphyCategory(dept: string | null, auctionTitle: string, lotTitle: string): string | null {282 const d = (dept ?? '').toLowerCase();283 const a = auctionTitle.toLowerCase();284 const t = lotTitle;285 if (/firearm|weapon|militaria|edged/.test(d) || /firearm|militaria/.test(a) || /\b(rifle|pistol|revolver|shotgun|carbine|ammunition|cartridge|bayonet)\b/i.test(t)) return null;286 if (/coin-op|coin op|arcade|slot machine|jukebox|vending/.test(d + a)) return /\b(slot machine|trade stimulator|casino)\b/i.test(t) ? 'casino_memorabilia' : /\b(jukebox|arcade|pinball)\b/i.test(t) ? 'arcade_pinball' : /\b(sign|tin|display|calendar|poster)\b/i.test(t) ? 'advertising' : 'vending_machines';287 if (/\bcoins?\b|currency|banknote|numismat/.test(d + a)) return /\b(note|bill|currency|dollar bill)\b/i.test(t) ? 'banknotes' : 'coins';288 if (/perfume|fragrance/.test(d + a)) return 'perfume';289 if (/petroliana|automobilia|gas|oil|soda|advertising/.test(d + a)) return /\b(sign|clock|thermometer|display|calendar|poster|globe)\b/i.test(t) || /advertising|petroliana|soda/.test(a) ? 'advertising' : 'automotive_memorabilia';290 if (/train/.test(d + a) || /\b(lionel|american flyer|marklin|märklin|ives|standard gauge|o gauge|ho scale)\b/i.test(t)) return 'model_trains';291 if (/doll/.test(d + a) && /\b(doll|bisque|barbie|steiff|teddy|bear)\b/i.test(t)) return 'dolls';292 if (/toy/.test(d + a)) return /\b(bank|cast iron)\b/i.test(t) ? 'vintage_toys' : 'vintage_toys';293 if (/sport/.test(d + a)) return sportsCategory(t);294 if (/jewel|watch/.test(d + a)) return /\b(watch|wristwatch|pocket watch)\b/i.test(t) ? 'other_watches' : 'jewelry';295 if (/fine|decorative|lamp|glass|art/.test(d + a)) {296 if (/\b(lamp|tiffany|handel|pairpoint)\b/i.test(t)) return 'antiques';297 if (/\b(perfume|atomizer|scent bottle)\b/i.test(t)) return 'perfume';298 if (/\b(vase|glass|paperweight|cameo glass|loetz|galle|gallé|daum|steuben|lalique|murano)\b/i.test(t)) return 'glass_crystal';299 if (/\b(porcelain|meissen|sevres|s[eè]vres|royal doulton|figurine|pottery|ceramic)\b/i.test(t)) return 'porcelain';300 if (/\b(sterling|silver)\b/i.test(t)) return 'silver';301 if (/\b(clock|regulator)\b/i.test(t)) return 'clocks';302 if (/\b(oil on canvas|painting|bronze|sculpture|lithograph|print)\b/i.test(t)) return 'art';303 return 'antiques';304 }305 if (/comic|pop culture|movie|music|entertainment/.test(d + a)) return popCultureCategory(t);306 return null;307}308309// ---------------------------------------------------------------------------------------------310// Builders for non-sale records311// ---------------------------------------------------------------------------------------------312export interface BaseInput {313 meta: ConnectorMeta;314 sourceUrl: string;315 externalId: string;316 rawTitle: string;317 attributes: AssetAttributes;318 imageUrls?: string[];319 description?: string | null;320 observedAt: Date;321 parserVersion: string;322 confidence?: number;323 grader?: string | null;324 grade?: string | null;325 condition?: string | null;326 conditionRaw?: string | null;327 completeness?: string | null;328}329330function base(i: BaseInput) {331 return {332 connectorId: i.meta.id,333 sourceId: i.meta.sourceId,334 sourceUrl: i.sourceUrl,335 externalId: i.externalId,336 rawTitle: i.rawTitle,337 description: i.description ?? null,338 imageUrls: i.imageUrls ?? [],339 attributes: i.attributes,340 grade: { grader: i.grader ?? null, grade: i.grade ?? null, qualifier: null, certificationNumber: null },341 condition: { condition: i.condition ?? null, conditionRaw: i.conditionRaw ?? null, completeness: i.completeness ?? null },342 observedAt: i.observedAt,343 confidence: i.confidence ?? 0.85,344 parserVersion: i.parserVersion,345 };346}347348export function makeListing(i: BaseInput & { price: number | null; currency: CurrencyCode | null; listingType?: NormalizedListing['listingType']; availability?: NormalizedListing['availability']; seller?: string | null; location?: string | null; quantity?: number | null; listedAt?: Date | null; endsAt?: Date | null; bidCount?: number | null }): NormalizedListing {349 return { kind: 'listing', ...base(i), listingType: i.listingType ?? 'fixed_price', price: i.price, currency: i.currency, seller: i.seller ?? null, sellerReputation: null, location: i.location ?? null, shippingCost: null, quantity: i.quantity ?? null, listedAt: i.listedAt ?? null, endsAt: i.endsAt ?? null, availability: i.availability ?? 'available', bidCount: i.bidCount ?? null };350}351352export function makeLot(i: BaseInput & { auctionHouse: string; auctionName?: string | null; lotNumber?: string | null; startsAt?: Date | null; endsAt?: Date | null; estimateLow?: number | null; estimateHigh?: number | null; currentBid?: number | null; currency: CurrencyCode | null; status?: NormalizedAuctionLot['status']; location?: string | null }): NormalizedAuctionLot {353 return { kind: 'auction_lot', ...base(i), auctionHouse: i.auctionHouse, auctionName: i.auctionName ?? null, lotNumber: i.lotNumber ?? null, startsAt: i.startsAt ?? null, endsAt: i.endsAt ?? null, estimateLow: i.estimateLow ?? null, estimateHigh: i.estimateHigh ?? null, currentBid: i.currentBid ?? null, currency: i.currency, status: i.status ?? 'unknown', location: i.location ?? null };354}355356export function makeCatalogItem(i: BaseInput & { releaseDate?: Date | null }): NormalizedCatalogItem {357 return { kind: 'catalog_item', ...base(i), releaseDate: i.releaseDate ?? null };358}359360/** Grade for sales/listings: only graded companies, never 'raw' as grader. */361export function gradeOf(title: string): { grader: string | null; grade: string | null } {362 const g = parseGradeFromTitle(title);363 if (!g.grader || g.grader === 'raw') return { grader: null, grade: null };364 let grade = g.grade;365 if (!grade) {366 // Houses often write the label between company and number ("SGC FR 1.5", "PSA VG-EX 4", "PSA NM-MT+ 8.5").367 const m = title.match(/\b(PSA|SGC|BGS|CGC|CBCS|TAG)\b[\s:-]*(?:[A-Z]{1,3}(?:[-/][A-Z]{1,3})?\+?\s*)?(\d{1,2}(?:\.\d)?)\b/i);368 if (m && Number(m[2]) <= 10) grade = m[2]!;369 }370 return { grader: g.grader, grade };371}372373/** AFA / CAS / UKG toy grades ("AFA 85", "AFA Qualified 80") kept as metadata, not graders (not in taxonomy). */374export function toyGrade(title: string): { company: string; grade: string } | null {375 const m = title.match(/\b(AFA|CAS|UKG)\s*(?:Qualified\s*)?(\d{2,3})/i);376 return m ? { company: m[1]!.toUpperCase(), grade: m[2]! } : null;377}378379/** Condition words from dealer labels → 'boxed_toys' scale (board games, trains, figures). */380export function dealerCondition(raw: string | null | undefined): { condition: string | null; completeness: string | null } {381 if (!raw) return { condition: null, completeness: null };382 const r = raw.toLowerCase();383 if (/\b(new|sealed|shrink|sw\b|mint in box|mib|misb|nib|moc)\b/.test(r)) return { condition: 'mint_in_box', completeness: 'sealed' };384 if (/\b(like new|ln\b|near mint|nm\b|mint\b)\b/.test(r)) return { condition: 'near_mint_box', completeness: 'boxed' };385 if (/\b(excellent|ex\b|very good|vg\b|vg\+|ex\+)\b/.test(r)) return { condition: 'boxed', completeness: 'boxed' };386 if (/\b(good|g\b|fair|played|used|restored)\b/.test(r)) return { condition: 'loose_complete', completeness: 'loose' };387 if (/\b(poor|damaged|parts|incomplete)\b/.test(r)) return { condition: 'loose', completeness: 'loose' };388 return { condition: null, completeness: null };389}390391export { lotAttributes, money };392