import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared'; import { fetchRawHtml, sneakerCategory, styleCodeFromText } from '../../api/_luxury-lib/index.js'; /** Flight Club — search grid from __NEXT_DATA__ (lowest ask + retail, USD). */ const BASE = 'https://www.flightclub.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ id: z.string(), name: z.string(), brand: z.string().nullable(), image: z.string().nullable(), price: z.number().nullable(), retail: z.number().nullable(), slug: z.string() }); export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), seed: z.string(), page: z.number(), currency: z.string(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; export function parseSearchPage(htmlText: string, url: string, seed: string, page: number): PagePayload | null { const nd = H.nextData(htmlText) as { props?: { pageProps?: { consumerSearchResult?: { data?: Array>; total?: number }; currency?: string; availableCurrencies?: unknown[] } } } | null; const r = nd?.props?.pageProps?.consumerSearchResult; if (!r?.data) return null; const items: z.infer[] = []; for (const d of r.data) { const id = d.id !== undefined ? String(d.id) : null; const slug = typeof d.slug === 'string' ? d.slug : null; if (!id || !slug) continue; const priceObj = d.price as { localizedValue?: number } | undefined; const retailObj = d.retailPrice as { localizedValue?: number } | undefined; items.push({ id, name: String(d.name ?? '').trim(), brand: d.brandName ? String(d.brandName) : null, image: d.pictureUrl ? String(d.pictureUrl) : null, price: typeof priceObj?.localizedValue === 'number' && priceObj.localizedValue > 0 ? priceObj.localizedValue : null, retail: typeof retailObj?.localizedValue === 'number' && retailObj.localizedValue > 0 ? retailObj.localizedValue : null, slug }); } return { kind: 'search_page', url, seed, page, currency: String(nd?.props?.pageProps?.currency ?? 'USD'), total: typeof r.total === 'number' ? r.total : null, items }; } export class FlightClubConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2); let count = 0; for (const seed of seeds) { for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/${seed}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); // __NEXT_DATA__ lives in a