import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared'; import { brandFromName, stripBrand } from '../_wlib/index.js'; /** * HiFiShark — aggregator of second-hand hi-fi listings (eBay, Subito, Kleinanzeigen, Audiogon, …). * The "For Sale" tab of a search page is server-rendered: title, price in the seller's currency, * marketplace, country, listing date and image. Sold/expired tabs load through endpoints that * robots.txt disallows, so only live asks are collected. */ const BASE = 'https://www.hifishark.com'; const PARSER_VERSION = '1.0.0'; export const RowSchema = z.object({ id: z.string(), href: z.string(), title: z.string(), priceText: z.string().nullable(), marketplace: z.string().nullable(), country: z.string().nullable(), countryIso: z.string().nullable(), firstSeen: z.string().nullable(), image: z.string().nullable(), }); export type Row = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), query: z.string(), forSaleCount: z.number().nullable(), rows: z.array(RowSchema) }); export type PagePayload = z.infer; export function parseSearchPage(htmlText: string, url: string, query: string): PagePayload { const $ = H.load(htmlText); const rows: Row[] = []; $('a.search-product-row').each((_, el) => { const a = $(el); const id = a.attr('id') ?? ''; const href = a.attr('href') ?? ''; const title = H.text(a.find('.search-product-title')); if (!id || !href || !title) return; const flag = a.find('.search-product-info img.flag').first(); const iso = (flag.attr('class') ?? '').match(/flag-([a-z]{2})/)?.[1] ?? null; rows.push({ id, href, title, priceText: H.text(a.find('.price strong')) ?? H.text(a.find('.price')), marketplace: H.text(a.find('.search-product-info .website'))?.replace(/\s+/g, ' ').trim() ?? null, country: flag.attr('title') ?? null, countryIso: iso ? iso.toUpperCase() : null, firstSeen: H.text(a.find('.first-seen')), image: a.find('.search-product-img img').attr('data-original') ?? null, }); }); const countText = $('#result-tabs .nav-link.active span').first().text().replace(/[()]/g, '').trim(); return { kind: 'search_page', url, query, forSaleCount: /^\d+$/.test(countText) ? Number(countText) : null, rows }; } export class HifisharkConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const queries = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.queries as string[] | undefined)) ?? []; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.queryIndex ?? 0) : 0; for (let qi = start; qi < queries.length; qi++) { const query = queries[qi]!; if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/search?q=${encodeURIComponent(query)}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', expect: ['title', 'price'], parse: (r) => (r.html && parseSearchPage(r.html, url, query).rows.length ? { title: 'ok', price: 1 } : null) }); const payload = res.success && res.html ? parseSearchPage(res.html, url, query) : null; if (!payload?.rows.length) { ctx.anomaly('page_fetch_failed', `${query}: ${res.error ?? res.httpStatus ?? 'no rows'}`); continue; } count++; yield { url, externalId: `q:${query}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.setCursor({ queryIndex: qi + 1 >= queries.length ? 0 : qi + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const brand = brandFromName(p.query); const model = stripBrand(p.query, brand); for (const r of p.rows) { const price = parsePrice(r.priceText ?? null); const listedAt = parseSourceDate(r.firstSeen); const attributes = AssetAttributesSchema.parse({ categorySlug: 'audio_equipment', brand, model, name: brand ? `${brand} ${model}` : p.query, identifiers: {}, metadata: { marketplace: r.marketplace, aggregator: 'HiFiShark', query: p.query }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: r.href.startsWith('http') ? r.href : `${BASE}${r.href}`, externalId: r.id, rawTitle: r.title, imageUrls: r.image ? [r.image] : [], attributes, observedAt: raw.fetchedAt, confidence: 0.6, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: price?.amount ?? null, currency: price?.currency ?? null, seller: r.marketplace, location: r.countryIso ?? r.country, listedAt, availability: 'available', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new HifisharkConnector(meta); }