spb/rareindex

1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';4import { brandFromName, stripBrand } from '../_wlib/index.js';56/**7 * HiFiShark — aggregator of second-hand hi-fi listings (eBay, Subito, Kleinanzeigen, Audiogon, …).8 * The "For Sale" tab of a search page is server-rendered: title, price in the seller's currency,9 * marketplace, country, listing date and image. Sold/expired tabs load through endpoints that10 * robots.txt disallows, so only live asks are collected.11 */12const BASE = 'https://www.hifishark.com';13const PARSER_VERSION = '1.0.0';1415export const RowSchema = z.object({16  id: z.string(),17  href: z.string(),18  title: z.string(),19  priceText: z.string().nullable(),20  marketplace: z.string().nullable(),21  country: z.string().nullable(),22  countryIso: z.string().nullable(),23  firstSeen: z.string().nullable(),24  image: z.string().nullable(),25});26export type Row = z.infer<typeof RowSchema>;27export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), query: z.string(), forSaleCount: z.number().nullable(), rows: z.array(RowSchema) });28export type PagePayload = z.infer<typeof PagePayloadSchema>;2930export function parseSearchPage(htmlText: string, url: string, query: string): PagePayload {31  const $ = H.load(htmlText);32  const rows: Row[] = [];33  $('a.search-product-row').each((_, el) => {34    const a = $(el);35    const id = a.attr('id') ?? '';36    const href = a.attr('href') ?? '';37    const title = H.text(a.find('.search-product-title'));38    if (!id || !href || !title) return;39    const flag = a.find('.search-product-info img.flag').first();40    const iso = (flag.attr('class') ?? '').match(/flag-([a-z]{2})/)?.[1] ?? null;41    rows.push({42      id,43      href,44      title,45      priceText: H.text(a.find('.price strong')) ?? H.text(a.find('.price')),46      marketplace: H.text(a.find('.search-product-info .website'))?.replace(/\s+/g, ' ').trim() ?? null,47      country: flag.attr('title') ?? null,48      countryIso: iso ? iso.toUpperCase() : null,49      firstSeen: H.text(a.find('.first-seen')),50      image: a.find('.search-product-img img').attr('data-original') ?? null,51    });52  });53  const countText = $('#result-tabs .nav-link.active span').first().text().replace(/[()]/g, '').trim();54  return { kind: 'search_page', url, query, forSaleCount: /^\d+$/.test(countText) ? Number(countText) : null, rows };55}5657export class HifisharkConnector extends BaseConnector {58  readonly version = '1.0.0';59  readonly parserVersion = PARSER_VERSION;60  protected override minIntervalMs = 2000;6162  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {63    const queries = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.queries as string[] | undefined)) ?? [];64    let count = 0;65    const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.queryIndex ?? 0) : 0;66    for (let qi = start; qi < queries.length; qi++) {67      const query = queries[qi]!;68      if (ctx.signal?.aborted || this.reached(ctx, count)) return;69      const url = `${BASE}/search?q=${encodeURIComponent(query)}`;70      await this.throttle();71      const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', expect: ['title', 'price'], parse: (r) => (r.html && parseSearchPage(r.html, url, query).rows.length ? { title: 'ok', price: 1 } : null) });72      const payload = res.success && res.html ? parseSearchPage(res.html, url, query) : null;73      if (!payload?.rows.length) {74        ctx.anomaly('page_fetch_failed', `${query}: ${res.error ?? res.httpStatus ?? 'no rows'}`);75        continue;76      }77      count++;78      yield { url, externalId: `q:${query}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };79      await ctx.setCursor({ queryIndex: qi + 1 >= queries.length ? 0 : qi + 1, updatedAt: new Date().toISOString() });80    }81  }8283  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {84    const p = PagePayloadSchema.parse(raw.payload);85    const out: NormalizedRecord[] = [];86    const brand = brandFromName(p.query);87    const model = stripBrand(p.query, brand);88    for (const r of p.rows) {89      const price = parsePrice(r.priceText ?? null);90      const listedAt = parseSourceDate(r.firstSeen);91      const attributes = AssetAttributesSchema.parse({92        categorySlug: 'audio_equipment',93        brand,94        model,95        name: brand ? `${brand} ${model}` : p.query,96        identifiers: {},97        metadata: { marketplace: r.marketplace, aggregator: 'HiFiShark', query: p.query },98      });99      out.push(100        NormalizedListingSchema.parse({101          kind: 'listing',102          connectorId: this.meta.id,103          sourceId: this.meta.sourceId,104          sourceUrl: r.href.startsWith('http') ? r.href : `${BASE}${r.href}`,105          externalId: r.id,106          rawTitle: r.title,107          imageUrls: r.image ? [r.image] : [],108          attributes,109          observedAt: raw.fetchedAt,110          confidence: 0.6,111          parserVersion: PARSER_VERSION,112          listingType: 'fixed_price',113          price: price?.amount ?? null,114          currency: price?.currency ?? null,115          seller: r.marketplace,116          location: r.countryIso ?? r.country,117          listedAt,118          availability: 'available',119        }),120      );121    }122    return out;123  }124}125126export default function createConnector(meta: ConnectorMeta) {127  return new HifisharkConnector(meta);128}129