TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import type { CrawlContext } from '@rareindex/connectors';3import { CurrencySchema, extractYear, type CurrencyCode } from '@rareindex/shared';45/**6 * Shared helpers for luxury / watch / sneaker marketplace connectors. Conventions match the7 * existing chrono24 (watches: brand/model/reference) and novelship (sneakers: style_code) connectors8 * so entity resolution merges records across sources.9 */1011// ---------- watches ----------12export const WATCH_BRAND_CATEGORY: Record<string, string> = {13 rolex: 'rolex',14 'patek philippe': 'patek_philippe',15 patek: 'patek_philippe',16 'audemars piguet': 'audemars_piguet',17 audemars: 'audemars_piguet',18 omega: 'omega',19};20export function watchCategory(brand: string | null | undefined): string {21 const b = (brand ?? '').toLowerCase().trim();22 for (const [k, v] of Object.entries(WATCH_BRAND_CATEGORY)) if (b === k || b.startsWith(k)) return v;23 return 'other_watches';24}25export const WATCH_BRANDS = ['Rolex', 'Patek Philippe', 'Audemars Piguet', 'Omega', 'Cartier', 'Tudor', 'Breitling', 'IWC', 'Jaeger-LeCoultre', 'A. Lange & Söhne', 'Vacheron Constantin', 'Richard Mille', 'Grand Seiko', 'Seiko', 'TAG Heuer', 'Panerai', 'Hublot', 'Zenith', 'Breguet', 'Blancpain', 'F.P. Journe', 'MB&F', 'De Bethune', 'Greubel Forsey', 'Urwerk', 'Chopard', 'Piaget', 'Bulgari', 'Bvlgari', 'Franck Muller', 'Girard-Perregaux', 'Glashütte Original', 'H. Moser & Cie', 'Jaquet Droz', 'Laurent Ferrier', 'Parmigiani', 'Roger Dubuis', 'Ulysse Nardin', 'Czapek', 'Longines', 'Tiffany & Co.', 'Hermès', 'Chanel', 'Louis Vuitton', 'Bell & Ross', 'Nomos', 'Oris', 'Sinn'];2627/** Same heuristics as chrono24: 116500LN, 126610LV, 5711/1A-010, 15400ST.OO.1220ST.01, 311.30.42.30.01.005, RM 011 */28export const WATCH_REF_RE = /\b(\d{4,6}[A-Z]{0,3}(?:\/\d[A-Z0-9]*)?(?:-\d{3})?|\d{5}[A-Z]{2}\.[A-Z]{2}\.\d{4}[A-Z]{2}\.\d{2}|\d{3}\.\d{2}\.\d{2}\.\d{2}\.\d{2}\.\d{3}|RM\s?\d{2,3}(?:-\d{2})?|W[A-Z0-9]{6,8}|[A-Z]{1,3}\d{3,5}[A-Z]{0,2}\.[A-Z0-9.]{3,})\b/;29export function watchReferenceFromText(text: string): string | null {30 return [...text.matchAll(new RegExp(WATCH_REF_RE.source, 'g'))].map((m) => m[1]!.replace(/\s+/g, ' ')).find((r) => !/^(18|19|20)\d{2}$/.test(r) && !/^\d{4}$/.test(r)) ?? null;31}32export function watchMaterial(text: string): string | null {33 const l = text.toLowerCase();34 if (/platinum/.test(l)) return 'platinum';35 if (/two[- ]tone|rolesor|steel (?:and|&) (?:yellow |rose |everose )?gold/.test(l)) return 'two-tone';36 if (/yellow gold|rose gold|everose|white gold|18k|18ct|gold/.test(l)) return 'gold';37 if (/titanium/.test(l)) return 'titanium';38 if (/ceramic/.test(l)) return 'ceramic';39 if (/steel|stainless|\bss\b/.test(l)) return 'steel';40 return null;41}42export function watchCompleteness(text: string): string | null {43 const l = text.toLowerCase();44 if (/full set|box (?:and|&|\/|\+) papers|box\/papers|complete set|box and card|box & card/.test(l)) return 'full_set';45 if (/papers|warranty card|guarantee/.test(l)) return 'papers_only';46 if (/\bbox\b/.test(l)) return 'box_only';47 return null;48}49export function watchConditionRaw(text: string): string | null {50 const l = text.toLowerCase();51 if (/unworn|brand new|\bnew\b|never worn/.test(l)) return 'Unworn';52 if (/like new|\bmint\b|excellent|pristine/.test(l)) return 'Excellent';53 if (/very good|good/.test(l)) return 'Good';54 if (/fair/.test(l)) return 'Fair';55 if (/pre-owned|used|preowned/.test(l)) return 'Pre-owned';56 return null;57}58export function caseSize(text: string): string | null {59 const m = text.match(/\b(\d{2}(?:\.\d)?)\s?mm\b/i);60 return m ? `${m[1]}mm` : null;61}6263// ---------- sneakers ----------64export function sneakerCategory(brand: string | null | undefined, name = ''): string {65 const b = `${brand ?? ''} ${name}`.toLowerCase();66 if (/jordan|nike/.test(b)) return 'nike_jordan';67 if (/adidas|yeezy/.test(b)) return 'adidas_yeezy';68 return 'new_balance_asics_other';69}70/** Style codes: FV5029-003, DZ5485-612, IE7264, GX1332, M990GL6, BB550STA, 1201A019-100, CT8012-170 */71export const STYLE_CODE_RE = /\b([A-Z]{1,2}\d{4,5}-\d{3}|\d{6}-\d{3}|[A-Z]{2}\d{4}|[A-Z]{1,3}\d{3,4}[A-Z]{1,4}\d{0,2}|\d{4}[A-Z]\d{3}-\d{3}|[A-Z]{2}\d{4}-\d{3})\b/;72export function styleCodeFromText(text: string): string | null {73 const m = text.toUpperCase().match(STYLE_CODE_RE);74 return m ? m[1]! : null;75}7677// ---------- handbags ----------78const BAG_MODELS = ['Birkin', 'Kelly', 'Constance', 'Lindy', 'Evelyne', 'Picotin', 'Garden Party', 'Bolide', 'Classic Flap', 'Classic Double Flap', 'Double Flap', 'Single Flap', '2.55', 'Reissue', 'Boy', 'Coco Handle', '19 Flap', 'Deauville', 'Gabrielle', 'Speedy', 'Neverfull', 'Alma', 'Keepall', 'Pochette Métis', 'Pochette Metis', 'Capucines', 'Twist', 'OnTheGo', 'Onthego', 'Multi Pochette', 'Noé', 'Petite Malle', 'Lady Dior', 'Saddle', 'Book Tote', 'Caro', 'Diorama', 'Jackie', 'Dionysus', 'Marmont', 'Bamboo', 'Horsebit', 'Saint Louis', 'Goyardine', 'Artois', 'Anjou', 'Baguette', 'Peekaboo', 'Cassette', 'Jodie', 'Pouch', 'Puzzle', 'Hammock', 'Galleria', 'Re-Edition', 'Cleo', 'Luggage', 'Triomphe', 'Loulou', 'Kate', 'Sac de Jour', 'Le 5 à 7'];79export function bagModel(title: string): string | null {80 const t = title.toLowerCase();81 for (const m of BAG_MODELS) if (t.includes(m.toLowerCase())) return m;82 return null;83}84export function bagSize(title: string): string | null {85 const m = title.match(/\b(\d{2})\b(?!\s?mm)/) ?? title.match(/\b(Mini|Small|Medium|Large|Jumbo|Maxi|PM|MM|GM|Nano|Micro)\b/i);86 return m ? m[1]! : null;87}88export const BAG_MATERIALS = ['Togo', 'Epsom', 'Clemence', 'Clémence', 'Swift', 'Box Calf', 'Chevre', 'Chèvre', 'Ostrich', 'Crocodile', 'Alligator', 'Lizard', 'Caviar', 'Lambskin', 'Calfskin', 'Patent', 'Canvas', 'Monogram', 'Damier', 'Epi', 'Vernis', 'Empreinte', 'Suede', 'Nylon', 'Denim', 'Tweed', 'Python', 'Goatskin', 'Goyardine', 'Raffia'];89export function bagMaterial(title: string): string | null {90 const t = title.toLowerCase();91 for (const m of BAG_MATERIALS) if (t.includes(m.toLowerCase())) return m;92 return null;93}94export function bagHardware(title: string): string | null {95 const m = title.match(/\b(Gold|Palladium|Silver|Rose Gold|Ruthenium|Brushed Gold|Light Gold|Aged Gold|Antique Gold|Gunmetal|Permabrass)\s+Hardware\b/i) ?? title.match(/\b(GHW|PHW|RGHW|SHW|BGHW)\b/);96 return m ? m[1]! : null;97}98const LUXURY_BAG_BRANDS = ['hermès', 'hermes', 'chanel', 'louis vuitton', 'dior', 'christian dior', 'gucci', 'goyard', 'fendi', 'bottega veneta', 'prada', 'celine', 'céline', 'loewe', 'saint laurent', 'ysl', 'balenciaga', 'givenchy', 'valentino', 'burberry', 'miu miu', 'chloé', 'chloe', 'jacquemus', 'the row'];99export function isLuxuryBagBrand(brand: string | null | undefined): boolean {100 return LUXURY_BAG_BRANDS.includes((brand ?? '').toLowerCase().trim());101}102103// ---------- Shopify ----------104export const ShopifyVariantSchema = z.object({105 id: z.number(),106 title: z.string(),107 price: z.string(),108 compare_at_price: z.string().nullable().optional(),109 available: z.boolean(),110 sku: z.string().nullable().optional(),111 option1: z.string().nullable().optional(),112 updated_at: z.string().optional(),113});114export const ShopifyProductSchema = z.object({115 id: z.number(),116 title: z.string(),117 handle: z.string(),118 body_html: z.string().nullable().optional(),119 published_at: z.string().nullable().optional(),120 created_at: z.string().optional(),121 updated_at: z.string().optional(),122 vendor: z.string().nullable().optional(),123 product_type: z.string().nullable().optional(),124 tags: z.array(z.string()).optional(),125 variants: z.array(ShopifyVariantSchema),126 images: z.array(z.object({ src: z.string() })).optional(),127 options: z.array(z.object({ name: z.string(), values: z.array(z.string()).optional() })).optional(),128});129export type ShopifyProduct = z.infer<typeof ShopifyProductSchema>;130131/** Trim a Shopify product to what normalize() needs (payloads stay compact). */132export function trimShopifyProduct(p: ShopifyProduct): ShopifyProduct {133 return {134 id: p.id,135 title: p.title,136 handle: p.handle,137 body_html: p.body_html ? p.body_html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 1200) : null,138 published_at: p.published_at ?? null,139 updated_at: p.updated_at,140 vendor: p.vendor ?? null,141 product_type: p.product_type ?? null,142 tags: (p.tags ?? []).filter((t) => !/^(updated-|amazon-|tiktok-)/.test(t)).slice(0, 40),143 variants: p.variants.slice(0, 20).map((v) => ({ id: v.id, title: v.title, price: v.price, compare_at_price: v.compare_at_price ?? null, available: v.available, sku: v.sku ?? null, option1: v.option1 ?? null, updated_at: v.updated_at })),144 images: (p.images ?? []).slice(0, 2).map((i) => ({ src: i.src })),145 options: (p.options ?? []).map((o) => ({ name: o.name })),146 };147}148149/** Fetch one page of a public Shopify products.json feed (plain HTTPS, no auth). */150export async function fetchShopifyPage(ctx: CrawlContext, base: string, path: string, page: number, limit = 250): Promise<{ products: ShopifyProduct[]; res: Awaited<ReturnType<CrawlContext['fetch']>> }> {151 const url = `${base}${path}?limit=${limit}&page=${page}`;152 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => {153 const j = r.json as { products?: Array<{ title?: string; variants?: Array<{ price?: string }> }> } | null;154 const f = j?.products?.[0];155 return f ? { title: f.title, price: f.variants?.[0]?.price } : { title: 'empty-page', price: 0 };156 } });157 const parsed = z.object({ products: z.array(ShopifyProductSchema) }).safeParse(res.json);158 return { products: parsed.success ? parsed.data.products : [], res };159}160161export function currencyOr(code: string | null | undefined, fallback: CurrencyCode): CurrencyCode {162 const c = CurrencySchema.safeParse((code ?? '').toUpperCase());163 return c.success ? c.data : fallback;164}165166export function yearFrom(text: string | null | undefined): number | null {167 return text ? extractYear(text) : null;168}169170export function moneyNumber(s: string | number | null | undefined): number | null {171 if (s === null || s === undefined) return null;172 const n = typeof s === 'number' ? s : Number(String(s).replace(/[^0-9.]/g, ''));173 return Number.isFinite(n) && n > 0 ? n : null;174}175176// ---------- Firecrawl rawHtml (scripts kept) ----------177import type { ExtractionResult } from '@rareindex/shared';178/**179 * The shared Firecrawl engine requests the `html` format, which strips <script> blocks — pages whose180 * data lives in __NEXT_DATA__ need `rawHtml`. This helper calls Firecrawl directly for rawHtml and181 * books the attempt into ctx.engineStats so cost/health accounting stays correct. Falls back to182 * ctx.fetch when no key is configured.183 */184export async function fetchRawHtml(ctx: CrawlContext, url: string, opts: { waitForMs?: number; timeoutMs?: number } = {}): Promise<ExtractionResult> {185 const key = process.env.FIRECRAWL_API_KEY;186 if (!key) return ctx.fetch(url, { minQuality: 0.2 });187 const started = Date.now();188 const stats = (ctx.engineStats.firecrawl ??= { attempts: 0, success: 0, credits: 0, ms: 0 });189 stats.attempts++;190 try {191 const res = await fetch('https://api.firecrawl.dev/v2/scrape', {192 method: 'POST',193 headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },194 body: JSON.stringify({ url, formats: ['rawHtml'], onlyMainContent: false, timeout: opts.timeoutMs ?? 60_000, ...(opts.waitForMs ? { waitFor: opts.waitForMs } : {}) }),195 });196 const text = await res.text();197 const json = JSON.parse(text) as { success?: boolean; data?: { rawHtml?: string; metadata?: { statusCode?: number; url?: string } }; error?: string; creditsUsed?: number };198 const status = json.data?.metadata?.statusCode ?? null;199 const ok = Boolean(res.ok && json.success && json.data?.rawHtml && (status === null || status < 400));200 stats.credits += json.creditsUsed ?? 1;201 stats.ms += Date.now() - started;202 if (ok) stats.success++;203 return { success: ok, engine: 'firecrawl', url, finalUrl: json.data?.metadata?.url ?? null, httpStatus: status, html: json.data?.rawHtml ?? null, markdown: null, json: null, qualityScore: ok ? 0.8 : 0, requiresReview: false, error: ok ? null : (json.error ?? `status ${status ?? res.status}`), costCredits: json.creditsUsed ?? 1, durationMs: Date.now() - started, fetchedAt: new Date() };204 } catch (err) {205 stats.ms += Date.now() - started;206 return { success: false, engine: 'firecrawl', url, finalUrl: null, httpStatus: null, html: null, markdown: null, json: null, qualityScore: 0, requiresReview: false, error: err instanceof Error ? err.message : String(err), costCredits: 0, durationMs: Date.now() - started, fetchedAt: new Date() };207 }208}209