spb/rareindex

1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';4import { sneakerCategory, styleCodeFromText } from '../../api/_luxury-lib/index.js';56/** Hypeboost — category grid (EUR lowest price) + product-page lookup (style code from JSON-LD). */7const BASE = 'https://hypeboost.com';8const PARSER_VERSION = '1.0.0';910export const ItemSchema = z.object({ id: z.string(), name: z.string(), brand: z.string().nullable(), category: z.string().nullable(), url: z.string(), image: z.string().nullable(), price: z.number(), currency: z.string(), sku: z.string().nullable() });11export const PagePayloadSchema = z.object({ kind: z.literal('grid_page'), url: z.string(), seed: z.string(), page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) });12export type PagePayload = z.infer<typeof PagePayloadSchema>;1314export function parseGridPage(htmlText: string, url: string, seed: string, page: number): PagePayload {15  const $ = H.load(htmlText);16  const items: z.infer<typeof ItemSchema>[] = [];17  $('.grid_item').each((_, el) => {18    const a = $(el).find('a[href*="/product/"]').first();19    const href = a.attr('href');20    const name = $(el).find('h3').first().text().replace(/\s+/g, ' ').trim();21    const priceText = $(el).find('.new_price').first().text();22    const parsed = parsePrice(priceText, 'EUR');23    const meta = $(el).find('[data-item-id]').first();24    if (!href || !name || !parsed) return;25    items.push({ id: meta.attr('data-item-id') ?? href, name, brand: meta.attr('data-item_brand') ?? null, category: meta.attr('data-item_category') ?? null, url: href.startsWith('http') ? href : `${BASE}${href}`, image: $(el).find('img').first().attr('data-src') ?? $(el).find('img').first().attr('src') ?? null, price: parsed.amount, currency: parsed.currency ?? 'EUR', sku: null });26  });27  const total = htmlText.match(/of\s+([\d,.]+)\s+results/)?.[1];28  return { kind: 'grid_page', url, seed, page, total: total ? Number(total.replace(/[,.]/g, '')) : null, items };29}3031export function parseProductPage(htmlText: string, url: string): PagePayload | null {32  const prod = H.jsonLd(htmlText, 'Product')[0];33  if (!prod) return null;34  const offers = (prod.offers as Record<string, unknown> | undefined) ?? {};35  const price = Number(offers.price ?? (offers as { lowPrice?: unknown }).lowPrice);36  if (!Number.isFinite(price) || price <= 0) return null;37  const img = prod.image;38  const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : null;39  return { kind: 'grid_page', url, seed: 'lookup', page: 1, total: null, items: [{ id: url.replace(/^.*\/product\//, ''), name: String(prod.name ?? ''), brand: brand || null, category: null, url, image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null, price, currency: String(offers.priceCurrency ?? 'EUR'), sku: prod.sku ? String(prod.sku) : null }] };40}4142export class HypeboostConnector extends BaseConnector {43  readonly version = '1.0.0';44  readonly parserVersion = PARSER_VERSION;45  protected override minIntervalMs = 2000;46  override readonly urlPatterns = [/^https?:\/\/(www\.)?hypeboost\.com\/[a-z]{2}\/product\/[a-z0-9-]+/i];4748  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {49    const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];50    const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);51    let count = 0;52    for (const seed of seeds) {53      let prevIds = '';54      for (let page = 1; page <= pages; page++) {55        if (ctx.signal?.aborted || this.reached(ctx, count)) return;56        const url = `${BASE}/en/category/sneakers/${seed}${page > 1 ? `?page=${page}` : ''}`;57        await this.throttle();58        const res = await ctx.fetch(url, { expect: ['title', 'price'], parse: (r) => {59          const first = r.html ? parseGridPage(r.html, url, seed, page).items[0] : undefined;60          return first ? { title: first.name, price: first.price } : null;61        } });62        const payload = res.success && res.html ? parseGridPage(res.html, url, seed, page) : null;63        if (!payload || payload.items.length === 0) {64          if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);65          break;66        }67        const ids = payload.items.map((i) => i.id).join(',');68        if (ids === prevIds) break; // pagination not honoured → same grid again69        prevIds = ids;70        count++;71        yield { url, externalId: `grid:${seed}:${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };72      }73    }74  }7576  async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {77    await this.throttle();78    const res = await ctx.fetch(url, { minQuality: 0.2 });79    const payload = res.success && res.html ? parseProductPage(res.html, url) : null;80    return payload ? [{ url, externalId: `product:${payload.items[0]!.id}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];81  }8283  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {84    const p = PagePayloadSchema.parse(raw.payload);85    const out: NormalizedRecord[] = [];86    for (const it of p.items) {87      const styleCode = it.sku ?? styleCodeFromText(it.name);88      const attributes = AssetAttributesSchema.parse({89        categorySlug: sneakerCategory(it.brand, it.name),90        brand: it.brand,91        series: it.category,92        name: it.name,93        identifiers: { ...(styleCode ? { style_code: styleCode } : {}), hypeboost_id: it.id },94      });95      const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };96      const cond = { condition: 'new', conditionRaw: 'Brand new (marketplace standard)', completeness: 'with_box' };97      out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${it.id}`, confidence: styleCode ? 0.8 : 0.6 }));98      out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `product:${it.id}:lowest_ask`, confidence: 0.7, condition: cond, listingType: 'ask', price: it.price, currency: it.currency === 'EUR' ? 'EUR' : 'EUR', seller: 'Hypeboost', location: 'NL', availability: 'available' }));99    }100    return out;101  }102}103104export default function createConnector(meta: ConnectorMeta) {105  return new HypeboostConnector(meta);106}107