TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { CurrencyCode, NormalizedRecord } from '@rareindex/shared';4import { amount, apolloRef, certFromTitle, hibidApolloState, isBundleTitle, isoDate, lotAttributes, makeSale, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js';5import { hibidCategory, isFirearm } from './categories.js';67const BASE = 'https://hibid.com';8const PARSER_VERSION = '1.0.0';9const PAST_PAGE_LENGTH = 25;10const ALLOWED_CURRENCIES = new Set<CurrencyCode>(['USD', 'CAD', 'GBP', 'EUR', 'AUD']);1112export const AuctioneerSchema = z.object({ id: z.string().nullable(), name: z.string(), city: z.string().nullable(), state: z.string().nullable(), country: z.string().nullable() });13export const AuctionSchema = z.object({14 id: z.string(),15 eventName: z.string(),16 url: z.string(),17 bidCloseDateTime: z.string().nullable(),18 eventDateEnd: z.string().nullable(),19 currency: z.string().nullable(),20 buyerPremium: z.string().nullable(),21 buyerPremiumRate: z.number().nullable(),22 auctioneer: AuctioneerSchema,23});24export const LotSchema = z.object({25 id: z.string(),26 lotNumber: z.string().nullable(),27 title: z.string(),28 description: z.string().nullable(),29 estimateText: z.string().nullable(),30 image: z.string().nullable(),31 categoryPath: z.string().nullable(),32 categoryName: z.string().nullable(),33 priceRealized: z.number().nullable(),34 quantitySold: z.number().nullable(),35 quantity: z.number().nullable(),36 bidCount: z.number().nullable(),37 isClosed: z.boolean(),38 url: z.string(),39});40export const PayloadSchema = z.object({ kind: z.literal('hibid_catalog_page'), auction: AuctionSchema, page: z.number(), totalCount: z.number().nullable(), lots: z.array(LotSchema) });41export type Payload = z.infer<typeof PayloadSchema>;42export type ParsedLot = z.infer<typeof LotSchema> & { mappable: boolean };4344type Cache = Record<string, Record<string, unknown>>;4546export function slugify(s: string): string {47 return s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 90);48}4950function str(v: unknown): string | null {51 return typeof v === 'string' && v.trim() ? v.trim() : null;52}5354function rootEntries(cache: Cache, prefix: string): Array<Record<string, unknown>> {55 const root = cache['ROOT_QUERY'] ?? {};56 return Object.entries(root)57 .filter(([k]) => k.startsWith(prefix))58 .map(([, v]) => apolloRef<Record<string, unknown>>(cache, v))59 .filter((v): v is Record<string, unknown> => v !== null);60}6162export interface PastAuctionRef {63 id: string;64 eventName: string;65 url: string;66 bidCloseDateTime: string | null;67 eventDateEnd: string | null;68 lotCount: number | null;69 status: string | null;70}7172/** /auctions/past?apage=N → archived auction references + paging. */73export function parsePastAuctions(html: string): { auctions: PastAuctionRef[]; pageNumber: number | null; totalCount: number | null; pageLength: number | null } | null {74 const cache = hibidApolloState(html);75 if (!cache) return null;76 const search = rootEntries(cache, 'auctionSearch(')[0];77 const paged = search ? apolloRef<Record<string, unknown>>(cache, search.pagedResults) : null;78 const results = Array.isArray(paged?.results) ? (paged!.results as unknown[]) : [];79 const auctions: PastAuctionRef[] = [];80 for (const r of results) {81 const match = apolloRef<Record<string, unknown>>(cache, r);82 const a = apolloRef<Record<string, unknown>>(cache, match?.auction ?? match);83 if (!a || typeof a.id !== 'number') continue;84 const name = str(a.eventName) ?? `Auction ${a.id}`;85 const state = apolloRef<Record<string, unknown>>(cache, a.auctionState);86 auctions.push({ id: String(a.id), eventName: name, url: `${BASE}/catalog/${a.id}/${slugify(name)}`, bidCloseDateTime: str(a.bidCloseDateTime), eventDateEnd: str(a.eventDateEnd), lotCount: typeof a.lotCount === 'number' ? a.lotCount : null, status: str(state?.auctionStatus) });87 }88 return { auctions, pageNumber: typeof paged?.pageNumber === 'number' ? paged.pageNumber : null, totalCount: typeof paged?.totalCount === 'number' ? paged.totalCount : null, pageLength: typeof paged?.pageLength === 'number' ? paged.pageLength : null };89}9091/** /catalog/<id>/<slug>?apage=N → auction header + every lot on the page (with a `mappable` flag). */92export function parseCatalog(html: string, page: number): { auction: Payload['auction']; lots: ParsedLot[]; pageNumber: number | null; totalCount: number | null; pageLength: number | null } | null {93 const cache = hibidApolloState(html);94 if (!cache) return null;95 const auctionRaw = rootEntries(cache, 'auction(')[0] ?? Object.entries(cache).find(([k]) => k.startsWith('Auction:'))?.[1] ?? null;96 if (!auctionRaw || typeof auctionRaw.id !== 'number') return null;97 const auctioneer = apolloRef<Record<string, unknown>>(cache, auctionRaw.auctioneer);98 const name = str(auctionRaw.eventName) ?? `Auction ${auctionRaw.id}`;99 const auction: Payload['auction'] = {100 id: String(auctionRaw.id),101 eventName: name,102 url: `${BASE}/catalog/${auctionRaw.id}/${slugify(name)}`,103 bidCloseDateTime: str(auctionRaw.bidCloseDateTime),104 eventDateEnd: str(auctionRaw.eventDateEnd),105 currency: str(auctionRaw.currencyAbbreviation),106 buyerPremium: str(auctionRaw.buyerPremium),107 buyerPremiumRate: typeof auctionRaw.buyerPremiumRate === 'number' ? auctionRaw.buyerPremiumRate : null,108 auctioneer: { id: auctioneer && auctioneer.id !== undefined ? String(auctioneer.id) : null, name: str(auctioneer?.name) ?? 'HiBid auctioneer', city: str(auctioneer?.city), state: str(auctioneer?.state), country: str(auctioneer?.country) },109 };110 const search = rootEntries(cache, 'lotSearch(')[0];111 const paged = search ? apolloRef<Record<string, unknown>>(cache, search.pagedResults) : null;112 const refs = Array.isArray(paged?.results) ? (paged!.results as unknown[]) : Object.keys(cache).filter((k) => k.startsWith('Lot:')).map((k) => ({ __ref: k }));113 const lots: ParsedLot[] = [];114 const seen = new Set<string>();115 for (const r of refs) {116 const l = apolloRef<Record<string, unknown>>(cache, r);117 if (!l || typeof l.id !== 'number' || seen.has(String(l.id))) continue;118 seen.add(String(l.id));119 const title = str(l.lead) ?? str((apolloRef<Record<string, unknown>>(cache, l.featuredPicture) ?? {}).description);120 if (!title) continue;121 const state = apolloRef<Record<string, unknown>>(cache, l.lotState) ?? {};122 const cats = (Array.isArray(l.category) ? l.category : []).map((c) => apolloRef<Record<string, unknown>>(cache, c)).filter((c): c is Record<string, unknown> => c !== null);123 // Most specific category first (longest fullCategory path).124 cats.sort((a, b) => String(b.fullCategory ?? '').length - String(a.fullCategory ?? '').length);125 const categoryPath = str(cats[0]?.fullCategory);126 const categoryName = str(cats[0]?.categoryName);127 const picture = apolloRef<Record<string, unknown>>(cache, l.featuredPicture);128 const description = str(l.description);129 const slug = hibidCategory(categoryPath, title);130 const mappable = slug !== null && !isFirearm(`${categoryPath ?? ''} ${title} ${description ?? ''}`);131 lots.push({132 id: String(l.id),133 lotNumber: str(l.lotNumber),134 title,135 description: description ? description.slice(0, 1500) : null,136 estimateText: str(l.estimate),137 image: str(picture?.fullSizeLocation) ?? str(picture?.hdThumbnailLocation),138 categoryPath,139 categoryName,140 priceRealized: amount(state.priceRealized),141 quantitySold: typeof state.quantitySold === 'number' ? state.quantitySold : null,142 quantity: typeof l.quantity === 'number' ? l.quantity : null,143 bidCount: typeof state.bidCount === 'number' ? state.bidCount : null,144 isClosed: state.isClosed === true || state.isArchived === true,145 url: `${BASE}/lot/${l.id}/${slugify(title)}`,146 mappable,147 });148 }149 return { auction, lots, pageNumber: typeof paged?.pageNumber === 'number' ? paged.pageNumber : null, totalCount: typeof paged?.totalCount === 'number' ? paged.totalCount : null, pageLength: typeof paged?.pageLength === 'number' ? paged.pageLength : null };150}151152/**153 * HiBid — public past catalogs of thousands of regional auctioneers. Plain HTTPS on hibid.com; the lot154 * data is read from the Apollo state the public Angular page embeds (`<script id="hibid-state">`).155 * Sales = closed lots whose auctioneer publishes the price realized (hammer; premium charged separately).156 */157export class HibidConnector extends BaseConnector {158 readonly version = '1.0.0';159 readonly parserVersion = PARSER_VERSION;160 protected override minIntervalMs = 3000;161162 private async html(ctx: CrawlContext, url: string): Promise<{ html: string | null; status: number | null; fetchedAt: Date }> {163 await this.throttle(url);164 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });165 if (!res.success || !res.html) {166 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);167 return { html: null, status: res.httpStatus, fetchedAt: res.fetchedAt };168 }169 return { html: res.html, status: res.httpStatus, fetchedAt: res.fetchedAt };170 }171172 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {173 const cfg = this.meta.config;174 const mode = ctx.options.mode;175 const pastPagesPerRun = mode === 'probe' ? 1 : Number(cfg.pastPagesPerRun ?? 3);176 const auctionsPerRun = mode === 'probe' ? 3 : Number(cfg.auctionsPerRun ?? 6);177 const pagesPerAuction = mode === 'probe' ? 1 : Number(cfg.pagesPerAuction ?? 10);178 const cursor = ctx.options.cursor ?? {};179 const doneAuctions = new Set<number>(Array.isArray(cursor.doneAuctions) ? (cursor.doneAuctions as number[]) : []);180 let pastPage = mode === 'backfill' ? Number(cursor.pastPage ?? 1) : 1;181 let filtered = 0;182 let rawCount = 0;183 let items = 0;184185 // Seeds (catalog URLs) or the past-auction list decide which catalogs to visit.186 const seeds = (ctx.options.seeds ?? []).map((s) => s.match(/\/catalog\/(\d+)/)?.[1]).filter((x): x is string => Boolean(x));187 const queue: PastAuctionRef[] = seeds.map((id) => ({ id, eventName: `Auction ${id}`, url: `${BASE}/catalog/${id}/x`, bidCloseDateTime: null, eventDateEnd: null, lotCount: null, status: null }));188 let totalPastPages: number | null = null;189 if (!seeds.length) {190 for (let i = 0; i < pastPagesPerRun; i++) {191 if (ctx.signal?.aborted) break;192 const url = `${BASE}/auctions/past${pastPage > 1 ? `?apage=${pastPage}` : ''}`;193 let r = await this.html(ctx, url);194 if (!r.html) break;195 let list = parsePastAuctions(r.html);196 if (list && list.auctions.length === 0 && list.pageNumber === null) {197 // The server occasionally renders the shell before the auction search resolves → one retry.198 ctx.log.warn({ url }, 'past-auction list rendered without results; retrying once');199 r = await this.html(ctx, `${url}${url.includes('?') ? '&' : '?'}apage=${pastPage}`);200 if (r.html) list = parsePastAuctions(r.html);201 }202 if (!list) {203 ctx.anomaly('parse_failure_page', `${url}: no hibid-state`);204 break;205 }206 if (list.totalCount !== null) totalPastPages = Math.max(1, Math.ceil(list.totalCount / (list.pageLength ?? PAST_PAGE_LENGTH)));207 if (list.auctions.length === 0) ctx.anomaly('pagination_failure', `${url}: past-auction list rendered without results (pageNumber ${list.pageNumber}, total ${list.totalCount})`);208 const fresh = list.auctions.filter((a) => !doneAuctions.has(Number(a.id)) && (a.status === null || a.status === 'ARCHIVED'));209 queue.push(...fresh);210 if (list.auctions.length === 0 || (totalPastPages !== null && pastPage >= totalPastPages)) {211 if (mode === 'backfill' && totalPastPages !== null && pastPage >= totalPastPages) {212 await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, done: true, updatedAt: new Date().toISOString() });213 pastPage = totalPastPages;214 }215 break;216 }217 pastPage++;218 if (queue.length >= auctionsPerRun) break;219 }220 }221222 let processed = 0;223 for (const a of queue) {224 if (ctx.signal?.aborted || processed >= auctionsPerRun || this.reached(ctx, rawCount)) break;225 processed++;226 const idNum = Number(a.id);227 for (let page = 1; page <= pagesPerAuction; page++) {228 if (ctx.signal?.aborted || this.reached(ctx, rawCount)) break;229 const url = `${a.url}${page > 1 ? `?apage=${page}` : ''}`;230 const r = await this.html(ctx, url);231 if (!r.html) break;232 const cat = parseCatalog(r.html, page);233 if (!cat) {234 ctx.anomaly('parse_failure_page', `${url}: no catalog state`);235 break;236 }237 if (cat.lots.length === 0) break;238 const mappable = cat.lots.filter((l) => l.mappable);239 filtered += cat.lots.length - mappable.length;240 if (page === 1 && mappable.length === 0) {241 doneAuctions.add(idNum); // non-collectible sale (equipment, real estate, household…) — skip the rest242 break;243 }244 if (mappable.length) {245 const payload: Payload = { kind: 'hibid_catalog_page', auction: cat.auction, page, totalCount: cat.totalCount, lots: mappable.map(({ mappable: _m, ...rest }) => rest) };246 rawCount++;247 items += mappable.length;248 yield { url, externalId: `catalog:${a.id}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt };249 }250 const pageLength = cat.pageLength ?? 100;251 if (cat.totalCount !== null && page * pageLength >= cat.totalCount) {252 doneAuctions.add(idNum);253 break;254 }255 if (page === pagesPerAuction) doneAuctions.add(idNum);256 }257 await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, updatedAt: new Date().toISOString() });258 await ctx.progress({ page: pastPage, totalPages: totalPastPages, itemsProcessed: items, cursor: { doneAuctions: [...doneAuctions].slice(-3000), pastPage } });259 }260 if (filtered) ctx.anomaly('filtered_non_collectible', `${filtered} lots outside the collectibles taxonomy skipped`);261 if (!seeds.length) await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-3000), pastPage, updatedAt: new Date().toISOString(), ...(mode === 'backfill' && totalPastPages !== null && pastPage >= totalPastPages ? { done: true } : {}) });262 }263264 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {265 const p = PayloadSchema.parse(raw.payload);266 const out: NormalizedRecord[] = [];267 const currency = p.auction.currency as CurrencyCode | null;268 if (!currency || !ALLOWED_CURRENCIES.has(currency)) return out;269 const saleDate = isoDate(p.auction.bidCloseDateTime) ?? isoDate(p.auction.eventDateEnd);270 if (!saleDate || saleDate.getTime() > Date.now() + 86_400_000) return out;271 const house = `${p.auction.auctioneer.name} (HiBid)`;272 const location = [p.auction.auctioneer.city, p.auction.auctioneer.state, p.auction.auctioneer.country].filter(Boolean).join(', ') || null;273 for (const l of p.lots) {274 if (!l.isClosed || !l.priceRealized) continue;275 const slug = hibidCategory(l.categoryPath, l.title);276 if (!slug || isFirearm(`${l.categoryPath ?? ''} ${l.title} ${l.description ?? ''}`)) continue;277 const g = saleGrade(l.title);278 const attributes = lotAttributes({279 categorySlug: slug,280 name: l.title,281 year: safeYear(l.title),282 identifiers: { hibid_lot_id: l.id, hibid_auction_id: p.auction.id },283 metadata: { auction_id: p.auction.id, event_name: p.auction.eventName, auctioneer_id: p.auction.auctioneer.id, estimate_text: l.estimateText, category_path: l.categoryPath, bid_count: l.bidCount, buyer_premium_text: p.auction.buyerPremium, buyer_premium_rate: p.auction.buyerPremiumRate, quantity_sold: l.quantitySold },284 });285 const sale = makeSale({ meta: this.meta, sourceUrl: l.url, externalId: l.id, rawTitle: l.title, description: l.description, attributes, price: l.priceRealized, currency, saleDate, buyerPremiumIncluded: false, auctionHouse: house, lotNumber: l.lotNumber, imageUrls: l.image ? [l.image] : [], location, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, isBundle: isBundleTitle(l.title) || (l.quantity ?? 1) > 1, confidence: g.grader ? 0.8 : 0.7 });286 sale.grade.qualifier = g.qualifier;287 sale.grade.certificationNumber = certFromTitle(l.title);288 sale.quantity = Math.max(1, l.quantitySold ?? l.quantity ?? 1);289 out.push(sale);290 }291 return out;292 }293}294295export default (meta: ConnectorMeta) => new HibidConnector(meta);296