import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; import { sneakerCategory, styleCodeFromText } from '../../api/_luxury-lib/index.js'; /** Hypeboost — category grid (EUR lowest price) + product-page lookup (style code from JSON-LD). */ const BASE = 'https://hypeboost.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ id: z.string(), name: z.string(), brand: z.string().nullable(), category: z.string().nullable(), url: z.string(), image: z.string().nullable(), price: z.number(), currency: z.string(), sku: z.string().nullable() }); export const PagePayloadSchema = z.object({ kind: z.literal('grid_page'), url: z.string(), seed: z.string(), page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; export function parseGridPage(htmlText: string, url: string, seed: string, page: number): PagePayload { const $ = H.load(htmlText); const items: z.infer[] = []; $('.grid_item').each((_, el) => { const a = $(el).find('a[href*="/product/"]').first(); const href = a.attr('href'); const name = $(el).find('h3').first().text().replace(/\s+/g, ' ').trim(); const priceText = $(el).find('.new_price').first().text(); const parsed = parsePrice(priceText, 'EUR'); const meta = $(el).find('[data-item-id]').first(); if (!href || !name || !parsed) return; items.push({ id: meta.attr('data-item-id') ?? href, name, brand: meta.attr('data-item_brand') ?? null, category: meta.attr('data-item_category') ?? null, url: href.startsWith('http') ? href : `${BASE}${href}`, image: $(el).find('img').first().attr('data-src') ?? $(el).find('img').first().attr('src') ?? null, price: parsed.amount, currency: parsed.currency ?? 'EUR', sku: null }); }); const total = htmlText.match(/of\s+([\d,.]+)\s+results/)?.[1]; return { kind: 'grid_page', url, seed, page, total: total ? Number(total.replace(/[,.]/g, '')) : null, items }; } export function parseProductPage(htmlText: string, url: string): PagePayload | null { const prod = H.jsonLd(htmlText, 'Product')[0]; if (!prod) return null; const offers = (prod.offers as Record | undefined) ?? {}; const price = Number(offers.price ?? (offers as { lowPrice?: unknown }).lowPrice); if (!Number.isFinite(price) || price <= 0) return null; const img = prod.image; const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : null; return { kind: 'grid_page', url, seed: 'lookup', page: 1, total: null, items: [{ id: url.replace(/^.*\/product\//, ''), name: String(prod.name ?? ''), brand: brand || null, category: null, url, image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null, price, currency: String(offers.priceCurrency ?? 'EUR'), sku: prod.sku ? String(prod.sku) : null }] }; } export class HypeboostConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; override readonly urlPatterns = [/^https?:\/\/(www\.)?hypeboost\.com\/[a-z]{2}\/product\/[a-z0-9-]+/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2); let count = 0; for (const seed of seeds) { let prevIds = ''; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/en/category/sneakers/${seed}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { expect: ['title', 'price'], parse: (r) => { const first = r.html ? parseGridPage(r.html, url, seed, page).items[0] : undefined; return first ? { title: first.name, price: first.price } : null; } }); const payload = res.success && res.html ? parseGridPage(res.html, url, seed, page) : null; if (!payload || payload.items.length === 0) { if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const ids = payload.items.map((i) => i.id).join(','); if (ids === prevIds) break; // pagination not honoured → same grid again prevIds = ids; count++; yield { url, externalId: `grid:${seed}:${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } } async lookup(url: string, ctx: CrawlContext): Promise { await this.throttle(); const res = await ctx.fetch(url, { minQuality: 0.2 }); const payload = res.success && res.html ? parseProductPage(res.html, url) : null; return payload ? [{ url, externalId: `product:${payload.items[0]!.id}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : []; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const it of p.items) { const styleCode = it.sku ?? styleCodeFromText(it.name); const attributes = AssetAttributesSchema.parse({ categorySlug: sneakerCategory(it.brand, it.name), brand: it.brand, series: it.category, name: it.name, identifiers: { ...(styleCode ? { style_code: styleCode } : {}), hypeboost_id: it.id }, }); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; const cond = { condition: 'new', conditionRaw: 'Brand new (marketplace standard)', completeness: 'with_box' }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${it.id}`, confidence: styleCode ? 0.8 : 0.6 })); out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `product:${it.id}:lowest_ask`, confidence: 0.7, condition: cond, listingType: 'ask', price: it.price, currency: it.currency === 'EUR' ? 'EUR' : 'EUR', seller: 'Hypeboost', location: 'NL', availability: 'available' })); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new HypeboostConnector(meta); }