spb/rareindex

1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';4import { fetchRawHtml, sneakerCategory, styleCodeFromText } from '../../api/_luxury-lib/index.js';56/** Flight Club — search grid from __NEXT_DATA__ (lowest ask + retail, USD). */7const BASE = 'https://www.flightclub.com';8const PARSER_VERSION = '1.0.0';910export const ItemSchema = z.object({ id: z.string(), name: z.string(), brand: z.string().nullable(), image: z.string().nullable(), price: z.number().nullable(), retail: z.number().nullable(), slug: z.string() });11export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: z.string(), page: z.number(), currency: z.string(), total: z.number().nullable(), items: z.array(ItemSchema) });12export type PagePayload = z.infer<typeof PagePayloadSchema>;1314export function parseSearchPage(htmlText: string, url: string, seed: string, page: number): PagePayload | null {15  const nd = H.nextData(htmlText) as { props?: { pageProps?: { consumerSearchResult?: { data?: Array<Record<string, unknown>>; total?: number }; currency?: string; availableCurrencies?: unknown[] } } } | null;16  const r = nd?.props?.pageProps?.consumerSearchResult;17  if (!r?.data) return null;18  const items: z.infer<typeof ItemSchema>[] = [];19  for (const d of r.data) {20    const id = d.id !== undefined ? String(d.id) : null;21    const slug = typeof d.slug === 'string' ? d.slug : null;22    if (!id || !slug) continue;23    const priceObj = d.price as { localizedValue?: number } | undefined;24    const retailObj = d.retailPrice as { localizedValue?: number } | undefined;25    items.push({ id, name: String(d.name ?? '').trim(), brand: d.brandName ? String(d.brandName) : null, image: d.pictureUrl ? String(d.pictureUrl) : null, price: typeof priceObj?.localizedValue === 'number' && priceObj.localizedValue > 0 ? priceObj.localizedValue : null, retail: typeof retailObj?.localizedValue === 'number' && retailObj.localizedValue > 0 ? retailObj.localizedValue : null, slug });26  }27  return { kind: 'search_page', url, seed, page, currency: String(nd?.props?.pageProps?.currency ?? 'USD'), total: typeof r.total === 'number' ? r.total : null, items };28}2930export class FlightClubConnector extends BaseConnector {31  readonly version = '1.0.0';32  readonly parserVersion = PARSER_VERSION;33  protected override minIntervalMs = 2000;3435  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {36    const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];37    const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);38    let count = 0;39    for (const seed of seeds) {40      for (let page = 1; page <= pages; page++) {41        if (ctx.signal?.aborted || this.reached(ctx, count)) return;42        const url = `${BASE}/${seed}${page > 1 ? `?page=${page}` : ''}`;43        await this.throttle();44        // __NEXT_DATA__ lives in a <script>; the shared engine's `html` format strips scripts, so ask Firecrawl for rawHtml.45        let res = await fetchRawHtml(ctx, url);46        let payload = res.success && res.html ? parseSearchPage(res.html, url, seed, page) : null;47        if (!payload) {48          res = await ctx.fetch(url, { engines: ['scrapfly'], expect: ['title', 'price', 'identifiers'], parse: (r) => {49            const first = r.html ? parseSearchPage(r.html, url, seed, page)?.items[0] : undefined;50            return first ? { title: first.name, price: first.price, identifiers: { slug: first.slug } } : null;51          } });52          payload = res.success && res.html ? parseSearchPage(res.html, url, seed, page) : null;53        }54        if (!payload || payload.items.length === 0) {55          if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);56          break;57        }58        count++;59        yield { url, externalId: `search:${seed}:${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };60      }61    }62  }6364  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {65    const p = PagePayloadSchema.parse(raw.payload);66    const out: NormalizedRecord[] = [];67    const currency = p.currency === 'USD' ? 'USD' : 'USD';68    for (const it of p.items) {69      const styleCode = styleCodeFromText(it.slug.split('-').slice(-2).join('-')) ?? styleCodeFromText(it.slug);70      const year = it.name.match(/\b(20\d{2}|19\d{2})\b\s*$/)?.[1];71      const attributes = AssetAttributesSchema.parse({72        categorySlug: sneakerCategory(it.brand, it.name),73        brand: it.brand,74        name: it.name.replace(/\s+\d{4}$/, '').trim(),75        year: year ? Number(year) : null,76        originalMsrp: it.retail,77        originalMsrpCurrency: it.retail ? currency : null,78        identifiers: { ...(styleCode ? { style_code: styleCode } : {}), flightclub_id: it.id },79        metadata: { slug: it.slug },80      });81      const sourceUrl = `${BASE}/${it.slug}`;82      const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };83      const cond = { condition: 'new', conditionRaw: 'New (Flight Club consignment)', completeness: 'with_box' };84      out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${it.id}`, confidence: 0.85 }));85      if (it.price) {86        out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${it.id}:low`, confidence: 0.7, condition: cond, priceKind: 'low', price: it.price, currency, observationDate: raw.fetchedAt }));87        out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: `product:${it.id}:lowest_ask`, confidence: 0.7, condition: cond, listingType: 'ask', price: it.price, currency, seller: 'Flight Club', location: 'US', availability: 'available' }));88      }89    }90    return out;91  }92}9394export default function createConnector(meta: ConnectorMeta) {95  return new FlightClubConnector(meta);96}97